Add pinned APFS FUSE source and guarded read-only NAS probe

这个提交包含在:
mac 于 2026-11-11 22:07:35 +08:00 提交
1 父节点 08e7da7f23
当前提交 737773eb76
111 个文件已更改
+26555 -3

没有与搜索条件相匹配的文件

+6 -3
查看文件
@@ -19,14 +19,17 @@
## 工程与恢复
- 本机:`/Users/mac/Documents/dayu-git/repositories/fnos-apfs`。
- 目标远程:`https://dayu.media/git/dayu/fnos-apfs.git`,分支 main;待创建并核验首个基线。
- 私有远程:`https://dayu.media/git/dayu/fnos-apfs.git`,分支 main;首个基线 `08e7da7f237c1a21aedad17b7a259726b8182060` 已推送并回读核验。
- `vendor/apfs-fuse` 保存完整源码快照,含 LZFSE 子模块内容;固定版本见 `UPSTREAM.json`。apfs-fuse `66b86bd525e8cb90f9012543be89b1f092b75cf3`,LZFSE `e634ca58b4821d9f3d560cdc6df5dec02ffc93fd`。
- `packaging/Dockerfile` 在固定 Debian 12 基础镜像中编译 Release;`scripts/build-driver.sh` 导出驱动;`scripts/validate-card.py` 按 PARTUUID + USB 序列号核对设备,临时设置内核只读,测试后卸载并恢复原标志。
- 验证程序检查挂载只读、样本复制 SHA-256、APFS 分区写计数和首 MiB 内容;这不是对整盘所有文件或全部 APFS 特性的兼容性保证。
- 不使用当前聊天原工作目录“大鱼调色”,不修改 NAS 网站生产源。
- 凭据在用户指定的本机安全位置,不进入工程。
## 待完成
1. 保存并核验远程工程基线。
2. 固定上游源码,在隔离的 Debian 12 容器内构建 Release 只读驱动。
1. 首次恢复克隆核验。
2. 在隔离的 Debian 12 容器内构建 Release 只读驱动。
3. 核验设备身份及驱动只读实现,测试列目录、读取与复制内容校验;不写源卡。
4. 清理临时挂载与容器,更新证据和远程检查点。
5. 应用界面、FPK、自动识别、系统文件管理集成及真实 SSD 性能尚未实现。
+10
查看文件
@@ -17,6 +17,16 @@
- [HANDOFF.md](HANDOFF.md):当前状态与下一步。
- 驱动来源:[sgan81/apfs-fuse](https://github.com/sgan81/apfs-fuse),只读 FUSE 实现;后续记录固定源码版本及许可证。
## 构建与验证
完整驱动快照及 LZFSE 源码位于 `vendor/`,版本见 `UPSTREAM.json`,许可证保留在各自目录内;不需要从上游重新下载源码。
在 x86_64 Linux + Docker 中,于工程根执行 `bash scripts/build-driver.sh`。仅容器内安装编译依赖;输出 `build/bin/apfs-fuse`、`apfsutil` 及依赖版本清单。NAS 本机需有 libfuse3、libstdc++、zlib、libbz2 的运行库。
验证入口为 `scripts/validate-card.py --partuuid <APFS分区PARTUUID> --serial <USB设备序列号> --workdir <工程根目录>`,需管理员运行。它会暂时挂载、读取最多三个 512 MiB 以内样本、将副本写入工程测试目录并比对 SHA-256,然后删除副本及卸载。不会尝试写 APFS 源盘。结果保存在工作目录 `result.json`。
SD 卡只适合验证基本功能,不能代表外置 SSD 的吞吐;隐藏索引文件的读取也不能当作素材导入测试。
## 恢复
目标远程:`https://dayu.media/git/dayu/fnos-apfs.git`,分支 `main`。
+12
查看文件
@@ -0,0 +1,12 @@
{
"apfs_fuse": {
"url": "https://github.com/sgan81/apfs-fuse.git",
"commit": "66b86bd525e8cb90f9012543be89b1f092b75cf3",
"license": "GPL-2.0-or-later"
},
"lzfse": {
"url": "https://github.com/lzfse/lzfse.git",
"commit": "e634ca58b4821d9f3d560cdc6df5dec02ffc93fd",
"license": "BSD-3-Clause"
}
}
+11
查看文件
@@ -0,0 +1,11 @@
FROM debian:12-slim@sha256:734a72b3d6142a71993b4bc580ca01d8ca687f77a2c422d96a961d3376b179bc
RUN apt-get update && apt-get install -y --no-install-recommends \
cmake g++ make libfuse3-dev zlib1g-dev libbz2-dev libattr1-dev \
&& rm -rf /var/lib/apt/lists/*
COPY vendor/apfs-fuse /src/apfs-fuse
RUN cmake -S /src/apfs-fuse -B /build -DUSE_FUSE3=ON \
&& cmake --build /build --target apfs-fuse apfsutil --parallel 2 \
&& mkdir /out \
&& cp /build/apfs-fuse /build/apfsutil /out/ \
&& strip /out/apfs-fuse /out/apfsutil \
&& dpkg-query -W > /out/build-packages.txt
+13
查看文件
@@ -0,0 +1,13 @@
#!/bin/bash
set -euo pipefail
cd "$(dirname "$0")/.."
image=dayu-fnos-apfs-validation:20261111
container=dayu-apfs-validation-export
docker build --progress=plain --tag "$image" --file packaging/Dockerfile .
docker create --name "$container" "$image" >/dev/null
trap 'docker rm "$container" >/dev/null' EXIT
mkdir -p build/bin
docker cp "$container:/out/." build/bin/
sha256sum build/bin/apfs-fuse build/bin/apfsutil
ldd build/bin/apfs-fuse
docker image inspect "$image" --format '{{.Id}}'
+150
查看文件
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""Read-only compatibility probe. Does not write or repair the source device."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import stat
import subprocess
import time
def run(*args, timeout=30):
return subprocess.check_output(args, text=True, timeout=timeout).strip()
def partition_writes(device):
fields = (Path('/sys/class/block') / device.name / 'stat').read_text().split()
return {'writes_completed': int(fields[4]), 'sectors_written': int(fields[6])}
def sample_hash(device):
with device.open('rb', buffering=0) as stream:
return hashlib.sha256(stream.read(1024 * 1024)).hexdigest()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--partuuid', required=True)
parser.add_argument('--serial', required=True)
parser.add_argument('--workdir', required=True, type=Path)
args = parser.parse_args()
if os.geteuid() != 0:
raise SystemExit('Run this hardware probe as root.')
work = args.workdir.resolve(strict=True)
device = (Path('/dev/disk/by-partuuid') / args.partuuid).resolve(strict=True)
if not stat.S_ISBLK(device.stat().st_mode):
raise SystemExit('Expected a block device.')
props = dict(line.split('=', 1) for line in run(
'udevadm', 'info', '--query=property', '--name=' + str(device)
).splitlines() if '=' in line)
if props.get('ID_BUS') != 'usb' or props.get('ID_SERIAL_SHORT') != args.serial:
raise SystemExit('USB identity mismatch; refusing to continue.')
if props.get('ID_PART_ENTRY_TYPE', '').lower() != '7c3457ef-0000-11aa-aa11-00306543ecac':
raise SystemExit('Partition is not declared APFS.')
with device.open('rb', buffering=0) as stream:
stream.seek(32)
if stream.read(4) != b'NXSB':
raise SystemExit('Missing APFS container signature.')
if subprocess.run(['findmnt', '-rn', '-S', str(device)], stdout=subprocess.DEVNULL).returncode == 0:
raise SystemExit('Source partition is already mounted.')
mount = work / 'mnt'
copies = work / 'read-check'
mount.mkdir(exist_ok=True)
if os.path.ismount(mount) or any(mount.iterdir()) or copies.exists():
raise SystemExit('Probe paths are occupied; refusing to reuse them.')
copies.mkdir()
report = {
'device': str(device), 'partuuid': args.partuuid,
'source_write_stats_before': partition_writes(device),
'source_header_sha256_before': sample_hash(device),
'files': [],
}
original_ro = run('blockdev', '--getro', str(device))
mounted = False
try:
subprocess.run(['blockdev', '--setro', str(device)], check=True)
report['block_read_only_during_test'] = run('blockdev', '--getro', str(device))
# The upstream implementation hard-codes read-only device opens and mounts.
subprocess.run([str(work / 'build/bin/apfs-fuse'), '-o', 'ro,nodev,nosuid,noexec',
str(device), str(mount)], check=True, timeout=30)
mounted = os.path.ismount(mount)
if not mounted:
raise RuntimeError('Driver exited without mounting.')
report['mount'] = json.loads(run('findmnt', '-J', '-M', str(mount),
'-o', 'SOURCE,FSTYPE,OPTIONS'))['filesystems'][0]
if 'ro' not in report['mount']['options'].split(','):
raise RuntimeError('Mount is not read-only.')
user_files = []
hidden_files = []
scanned = 0
for base, dirs, files in os.walk(mount, followlinks=False):
dirs.sort()
for name in sorted(files):
path = Path(base) / name
info = path.lstat()
scanned += 1
if stat.S_ISREG(info.st_mode) and 0 < info.st_size <= 512 * 1024**2:
relative = path.relative_to(mount)
hidden = any(part.startswith('.') for part in relative.parts)
(hidden_files if hidden else user_files).append((path, info.st_size))
if scanned >= 10000:
break
if scanned >= 10000:
break
report['scanned_files'] = scanned
report['eligible_user_files'] = len(user_files)
report['eligible_hidden_files'] = len(hidden_files)
selected = sorted(user_files or hidden_files, key=lambda item: item[1], reverse=True)[:3]
report['test_file_kind'] = 'user_files' if user_files else 'hidden_metadata_only'
for index, (source, expected_size) in enumerate(selected, 1):
target = copies / ('sample-' + str(index))
source_hash = hashlib.sha256()
started = time.monotonic()
read_bytes = 0
with source.open('rb') as inp, target.open('xb') as out:
while chunk := inp.read(1024 * 1024):
out.write(chunk)
source_hash.update(chunk)
read_bytes += len(chunk)
out.flush()
os.fsync(out.fileno())
elapsed = time.monotonic() - started
destination_hash = hashlib.sha256()
with target.open('rb') as stream:
while chunk := stream.read(1024 * 1024):
destination_hash.update(chunk)
if read_bytes != expected_size or source_hash.digest() != destination_hash.digest():
raise RuntimeError('Copied content did not match.')
report['files'].append({'index': index, 'bytes': read_bytes,
'sha256': source_hash.hexdigest(), 'copy_matches': True,
'elapsed_seconds': round(elapsed, 6),
'copy_MB_per_second': round(read_bytes / elapsed / 1e6, 3)})
report['status'] = 'passed' if selected else 'mounted_but_no_sample_files'
except Exception as error:
report['status'] = 'failed'
report['error'] = str(error)
raise
finally:
if mounted or os.path.ismount(mount):
subprocess.run(['umount', str(mount)], check=True, timeout=30)
if original_ro == '0':
subprocess.run(['blockdev', '--setrw', str(device)], check=True)
report['source_write_stats_after'] = partition_writes(device)
report['source_header_sha256_after'] = sample_hash(device)
report['source_writes_unchanged'] = report['source_write_stats_before'] == report['source_write_stats_after']
report['source_header_unchanged'] = report['source_header_sha256_before'] == report['source_header_sha256_after']
report['unmounted_after_test'] = not os.path.ismount(mount)
report['block_read_only_restored'] = run('blockdev', '--getro', str(device)) == original_ro
(work / 'result.json').write_text(json.dumps(report, indent=2) + '\n')
# Only this probe's destination copies are removed; source files remain untouched.
shutil.rmtree(copies)
print(json.dumps(report, indent=2))
if not report['source_writes_unchanged'] or not report['source_header_unchanged']:
raise SystemExit('Unexpected source changes detected.')
if __name__ == '__main__':
main()
+63
查看文件
@@ -0,0 +1,63 @@
###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
* text=auto
###############################################################################
# Set default behavior for command prompt diff.
#
# This is need for earlier builds of msysgit that does not have it on by
# default for csharp files.
# Note: This is only used by command line
###############################################################################
#*.cs diff=csharp
###############################################################################
# Set the merge driver for project and solution files
#
# Merging from the command prompt will add diff markers to the files if there
# are conflicts (Merging from VS is not affected by the settings below, in VS
# the diff markers are never inserted). Diff markers may cause the following
# file extensions to fail to load in VS. An alternative would be to treat
# these files as binary and thus will always conflict and require user
# intervention with every merge. To do so, just uncomment the entries below
###############################################################################
#*.sln merge=binary
#*.csproj merge=binary
#*.vbproj merge=binary
#*.vcxproj merge=binary
#*.vcproj merge=binary
#*.dbproj merge=binary
#*.fsproj merge=binary
#*.lsproj merge=binary
#*.wixproj merge=binary
#*.modelproj merge=binary
#*.sqlproj merge=binary
#*.wwaproj merge=binary
###############################################################################
# behavior for image files
#
# image files are treated as binary by default.
###############################################################################
#*.jpg binary
#*.png binary
#*.gif binary
###############################################################################
# diff behavior for common document formats
#
# Convert binary document formats to text before diffing them. This feature
# is only available from the command line. Turn it on by uncommenting the
# entries below.
###############################################################################
#*.doc diff=astextplain
#*.DOC diff=astextplain
#*.docx diff=astextplain
#*.DOCX diff=astextplain
#*.dot diff=astextplain
#*.DOT diff=astextplain
#*.pdf diff=astextplain
#*.PDF diff=astextplain
#*.rtf diff=astextplain
#*.RTF diff=astextplain
+275
查看文件
@@ -0,0 +1,275 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# DNX
project.lock.json
project.fragment.lock.json
artifacts/
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Comment the next line if you want to checkin your web deploy settings
# but database connection strings (with potential passwords) will be unencrypted
#*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignoreable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
# CodeRush
.cr/
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Own Files
Data/
# Linux
build/
.kdev4/
data/
doc/
# Mac
.DS_Store
xcode/
+3
查看文件
@@ -0,0 +1,3 @@
[submodule "3rdparty/lzfse"]
path = 3rdparty/lzfse
url = https://github.com/lzfse/lzfse.git
@@ -0,0 +1,6 @@
*~
.DS_Store
**/project.xcworkspace/
**/xcuserdata/
build/
+158
查看文件
@@ -0,0 +1,158 @@
language: c
sudo: false
matrix:
include:
###
## Linux builds using various versions of GCC.
###
- os: linux
env: C_COMPILER=gcc-6
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-6
- os: linux
env: C_COMPILER=gcc-5
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
- os: linux
env: C_COMPILER=gcc-4.9
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.9
- os: linux
env: C_COMPILER=gcc-4.8
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.8
- os: linux
env: C_COMPILER=gcc-4.7
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.7
- os: linux
env: C_COMPILER=gcc-4.6
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.6
- os: linux
env: C_COMPILER=gcc-4.5
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.5
- os: linux
env: C_COMPILER=gcc-4.4
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.4
###
## clang on Linux
###
- os: linux
env: C_COMPILER=clang-3.8
addons:
apt:
sources:
- llvm-toolchain-precise-3.8
- ubuntu-toolchain-r-test
packages:
- clang-3.8
- os: linux
env: C_COMPILER=clang-3.7
addons:
apt:
sources:
- llvm-toolchain-precise-3.7
- ubuntu-toolchain-r-test
packages:
- clang-3.7
- os: linux
env: C_COMPILER=clang-3.6
addons:
apt:
sources:
- llvm-toolchain-precise-3.6
- ubuntu-toolchain-r-test
packages:
- clang-3.6
- os: linux
env: C_COMPILER=clang-3.5
addons:
apt:
sources:
- llvm-toolchain-precise-3.5
- ubuntu-toolchain-r-test
packages:
- clang-3.5
###
## Sanitizers
###
- os: linux
env: C_COMPILER=clang-3.8 SANITIZER=address
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.8
packages:
- clang-3.8
- os: linux
env: C_COMPILER=clang-3.8 SANITIZER=thread
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.8
packages:
- clang-3.8
- os: linux
env: C_COMPILER=clang-3.8 SANITIZER=undefined CFLAGS="-fno-sanitize-recover=undefined,integer"
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.8
packages:
- clang-3.8
before_install:
###
## If we use the matrix to set CC/CXX Travis, overwrites the values,
## so instead we use C/CXX_COMPILER, then copy the values to CC/CXX
## here (after Travis has set CC/CXX).
###
- if [ -n "${C_COMPILER}" ]; then export CC="${C_COMPILER}"; fi
- mkdir build
- cd build
script:
- cmake --version
- cmake .. -DCMAKE_C_FLAGS="${CFLAGS}" -DENABLE_SANITIZER="${SANITIZER}"
- make VERBOSE=1
- ctest -V
+137
查看文件
@@ -0,0 +1,137 @@
project(lzfse C)
cmake_minimum_required(VERSION 2.8.6)
include(CheckCCompilerFlag)
# If LZFSE is being bundled in another project, we don't want to
# install anything. However, we want to let people override this, so
# we'll use the LZFSE_BUNDLE_MODE variable to let them do that; just
# set it to OFF in your project before you add_subdirectory(lzfse).
get_directory_property(LZFSE_PARENT_DIRECTORY PARENT_DIRECTORY)
if("${LZFSE_BUNDLE_MODE}" STREQUAL "")
# Bundled mode hasn't been set one way or the other, set the default
# depending on whether or not we are the top-level project.
if(LZFSE_PARENT_DIRECTORY)
set(LZFSE_BUNDLE_MODE ON)
else()
set(LZFSE_BUNDLE_MODE OFF)
endif(LZFSE_PARENT_DIRECTORY)
endif()
mark_as_advanced(LZFSE_BUNDLE_MODE)
if(NOT LZFSE_BUNDLE_MODE)
option(BUILD_SHARED_LIBS "Build shared libraries" ON)
else()
set(BUILD_SHARED_LIBS OFF)
endif()
if (CMAKE_VERSION VERSION_GREATER 3.2)
cmake_policy (SET CMP0063 NEW)
endif ()
# Compiler flags
function(lzfse_add_compiler_flags target)
set (flags ${ARGV})
list (REMOVE_AT flags 0)
foreach (FLAG ${flags})
if(CMAKE_C_COMPILER_ID STREQUAL GNU)
# Because https://gcc.gnu.org/wiki/FAQ#wnowarning
string(REGEX REPLACE "\\-Wno\\-(.+)" "-W\\1" flag_to_test "${FLAG}")
else()
set (flag_to_test ${FLAG})
endif()
string(REGEX REPLACE "[^a-zA-Z0-9]+" "_" test_name "CFLAG_${flag_to_test}")
check_c_compiler_flag("${flag_to_test}" "${test_name}")
if(${${test_name}})
set_property(TARGET "${target}" APPEND_STRING PROPERTY COMPILE_FLAGS " ${FLAG}")
endif()
endforeach()
endfunction()
if (ENABLE_SANITIZER)
set(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -fsanitize=${ENABLE_SANITIZER}")
set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -fsanitize=${ENABLE_SANITIZER}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=${ENABLE_SANITIZER}")
endif ()
add_library(lzfse
src/lzfse_decode.c
src/lzfse_decode_base.c
src/lzfse_encode.c
src/lzfse_encode_base.c
src/lzfse_fse.c
src/lzvn_decode_base.c
src/lzvn_encode_base.c)
lzfse_add_compiler_flags(lzfse -Wall -Wno-unknown-pragmas -Wno-unused-variable)
add_executable(lzfse_cli
src/lzfse_main.c)
target_link_libraries(lzfse_cli lzfse)
set_target_properties(lzfse_cli PROPERTIES OUTPUT_NAME lzfse)
lzfse_add_compiler_flags(lzfse_cli -Wall -Wno-unknown-pragmas -Wno-unused-variable)
if(CMAKE_VERSION VERSION_LESS 3.1 OR CMAKE_C_COMPLIER_ID STREQUAL "Intel")
lzfse_add_compiler_flags(lzfse -std=c99)
lzfse_add_compiler_flags(lzfse_cli -std=c99)
else()
set_property(TARGET lzfse PROPERTY C_STANDARD 99)
set_property(TARGET lzfse_cli PROPERTY C_STANDARD 99)
endif()
set_target_properties(lzfse PROPERTIES
POSITION_INDEPENDENT_CODE TRUE
C_VISIBILITY_PRESET hidden
INTERPROCEDURAL_OPTIMIZATION TRUE)
if(BUILD_SHARED_LIBS)
set_property(TARGET lzfse APPEND PROPERTY COMPILE_DEFINITIONS LZFSE_DLL LZFSE_DLL_EXPORTS)
set_property(TARGET lzfse APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS LZFSE_DLL)
endif()
# Installation
if(NOT LZFSE_BUNDLE_MODE)
include(GNUInstallDirs)
install(TARGETS lzfse lzfse_cli
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
install(FILES src/lzfse.h DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
endif()
# Tests
# If we're targeting Windows but not running on Windows, we need Wine
# to run the tests...
if(NOT LZFSE_DISABLE_TESTS)
if(WIN32 AND NOT CMAKE_HOST_WIN32)
find_program(LZFSE_WINE NAMES wine)
if(NOT LZFSE_WINE)
message(STATUS "wine not found, disabling tests")
set(LZFSE_DISABLE_TESTS TRUE)
endif()
endif()
endif()
if(NOT LZFSE_DISABLE_TESTS)
include(CTest)
enable_testing()
file(GLOB_RECURSE
ROUNDTRIP_INPUTS
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
src/*)
foreach(INPUT ${ROUNDTRIP_INPUTS})
add_test(NAME "${LZFSE_TEST_PREFIX}roundtrip/${INPUT}"
COMMAND "${CMAKE_COMMAND}"
-DLZFSE_WRAPPER=${LZFSE_WINE}
-DLZFSE_CLI=$<TARGET_FILE:lzfse_cli>
-DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/${INPUT}
-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/round-trip.cmake)
endforeach()
endif()
+27
查看文件
@@ -0,0 +1,27 @@
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
OF SUCH DAMAGE.
+64
查看文件
@@ -0,0 +1,64 @@
# Copyright (c) 2015-2016, Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
# following disclaimer in the documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or
# promote products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
INSTALL_PREFIX ?= /usr/local
BUILD_DIR := ./build
BIN_DIR := $(BUILD_DIR)/bin
OBJ_DIR := $(BUILD_DIR)/obj
LZFSE_LIB := $(BIN_DIR)/liblzfse.a
LZFSE_CMD := $(BIN_DIR)/lzfse
LIB_OBJS := $(OBJ_DIR)/lzfse_encode.o $(OBJ_DIR)/lzfse_decode.o \
$(OBJ_DIR)/lzfse_encode_base.o $(OBJ_DIR)/lzfse_decode_base.o \
$(OBJ_DIR)/lzvn_encode_base.o $(OBJ_DIR)/lzvn_decode_base.o \
$(OBJ_DIR)/lzfse_fse.o
CMD_OBJS := $(OBJ_DIR)/lzfse_main.o
OBJS := $(LIB_OBJS) $(CMD_OBJS)
CFLAGS := -Os -Wall -Wno-unknown-pragmas -Wno-unused-variable -DNDEBUG -D_POSIX_C_SOURCE -std=c99 -fvisibility=hidden
all: $(LZFSE_LIB) $(LZFSE_CMD) $(OBJS)
install: $(LZFSE_LIB) $(LZFSE_CMD)
@[ -d $(INSTALL_PREFIX)/include ] || mkdir -p $(INSTALL_PREFIX)/include
@[ -d $(INSTALL_PREFIX)/lib ] || mkdir -p $(INSTALL_PREFIX)/lib
@[ -d $(INSTALL_PREFIX)/bin ] || mkdir -p $(INSTALL_PREFIX)/bin
install ./src/lzfse.h $(INSTALL_PREFIX)/include/lzfse.h
install $(LZFSE_LIB) $(INSTALL_PREFIX)/lib/liblzfse.a
install $(LZFSE_CMD) $(INSTALL_PREFIX)/bin/lzfse
$(LZFSE_LIB): $(LIB_OBJS)
@[ -d $(BIN_DIR) ] || mkdir -p $(BIN_DIR)
$(LD) -r -o $(OBJ_DIR)/liblzfse_master.o $(LIB_OBJS)
ar rvs $@ $(OBJ_DIR)/liblzfse_master.o
$(LZFSE_CMD): $(CMD_OBJS) $(LZFSE_LIB)
@[ -d $(BIN_DIR) ] || mkdir -p $(BIN_DIR)
$(CC) $(CFLAGS) -o $@ $(CMD_OBJS) $(LZFSE_LIB)
clean:
/bin/rm -rf $(BUILD_DIR)
$(OBJ_DIR)/%.o: src/%.c
@[ -d $(OBJ_DIR) ] || mkdir -p $(OBJ_DIR)
$(CC) $(CFLAGS) -c $< -o $@
+73
查看文件
@@ -0,0 +1,73 @@
LZFSE
=====
This is a reference C implementation of the LZFSE compressor introduced in the
[Compression library](https://developer.apple.com/library/mac/documentation/Performance/Reference/Compression/index.html) with OS X 10.11 and iOS 9.
LZFSE is a Lempel-Ziv style data compression algorithm using Finite State Entropy coding.
It targets similar compression rates at higher compression and decompression speed compared to deflate using zlib.
Files
-----
README.md This file ;-)
Makefile Linux / macOS Makefile
lzfse.xcodeproj Xcode project
src/lzfse.h Main LZFSE header
src/lzfse_tunables.h LZFSE encoder configuration
src/lzfse_internal.h LZFSE internal header
src/lzfse_decode.c LZFSE decoder API entry point
src/lzfse_encode.c LZFSE encoder API entry point
src/lzfse_decode_base.c LZFSE decoder internal functions
src/lzfse_encode_base.c LZFSE encoder internal functions
src/lzfse_encode_tables.h LZFSE encoder tables
src/lzfse_fse.h FSE entropy encoder/decoder header
src/lzfse_fse.c FSE entropy encoder/decoder functions
src/lzvn_decode_base.h LZVN decoder
src/lzvn_decode_base.c
src/lzvn_encode_base.h LZVN encoder
src/lzvn_encode_base.c
src/lzfse_main.c Command line tool
Building on OS X
----------------
$ xcodebuild install DSTROOT=/tmp/lzfse.dst
Produces the following files in `/tmp/lzfse.dst`:
usr/local/bin/lzfse command line tool
usr/local/include/lzfse.h LZFSE library header
usr/local/lib/liblzfse.a LZFSE library
Building on Linux
-----------------
Tested on Ubuntu 15.10 with gcc 5.2.1 and clang 3.6.2. Should work on any recent distribution.
$ make install INSTALL_PREFIX=/tmp/lzfse.dst/usr/local
Produces the following files in `/tmp/lzfse.dst`:
usr/local/bin/lzfse command line tool
usr/local/include/lzfse.h LZFSE library header
usr/local/lib/liblzfse.a LZFSE library
Building with cmake
-------------------
$ mkdir build
$ cd build
$ cmake ..
$ make install
Installs the header, library, and command line tool in `/usr/local`.
Bindings
--------
Python: [dimkr/pylzfse](https://github.com/dimkr/pylzfse)
+20
查看文件
@@ -0,0 +1,20 @@
environment:
matrix:
- GENERATOR: Visual Studio 14 2015 Win64
- GENERATOR: Visual Studio 14 2015
- GENERATOR: Visual Studio 12 2013 Win64
- GENERATOR: Visual Studio 12 2013
before_build:
- ps: |
mkdir build
cd build
cmake -G "$env:GENERATOR" ..
build_script:
- ps: |
cmake --build . --config Debug
test_script:
- ps: |
ctest --output-on-failure --interactive-debug-mode 0 -C Debug
+517
查看文件
@@ -0,0 +1,517 @@
// !$*UTF8*$!
{
archiveVersion = 1;
classes = {
};
objectVersion = 46;
objects = {
/* Begin PBXAggregateTarget section */
0B1AF7D91B8D0F0B005BF60D /* all */ = {
isa = PBXAggregateTarget;
buildConfigurationList = 0B1AF7DA1B8D0F0B005BF60D /* Build configuration list for PBXAggregateTarget "all" */;
buildPhases = (
);
dependencies = (
0B1AF7DE1B8D0F16005BF60D /* PBXTargetDependency */,
0B1AF7E01B8D0F16005BF60D /* PBXTargetDependency */,
);
name = all;
productName = all;
};
/* End PBXAggregateTarget section */
/* Begin PBXBuildFile section */
0B1AF7C01B8CF8A2005BF60D /* lzvn_decode_base.c in Sources */ = {isa = PBXBuildFile; fileRef = 0B1AF7B71B8CF8A2005BF60D /* lzvn_decode_base.c */; };
0B1AF7C41B8CF8A2005BF60D /* lzvn_encode_base.c in Sources */ = {isa = PBXBuildFile; fileRef = 0B1AF7BB1B8CF8A2005BF60D /* lzvn_encode_base.c */; };
0B1AF7E31B8D0F65005BF60D /* lzfse_main.c in Sources */ = {isa = PBXBuildFile; fileRef = 0B1AF7E11B8D0F59005BF60D /* lzfse_main.c */; };
0B1AF7E41B8D1EFD005BF60D /* liblzfse.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 0B57822C1B8BC12F002F950F /* liblzfse.a */; };
0B57823B1B8BC151002F950F /* lzfse_decode_base.c in Sources */ = {isa = PBXBuildFile; fileRef = 0B5782321B8BC151002F950F /* lzfse_decode_base.c */; };
0B57823D1B8BC151002F950F /* lzfse_decode.c in Sources */ = {isa = PBXBuildFile; fileRef = 0B5782341B8BC151002F950F /* lzfse_decode.c */; };
0B57823E1B8BC151002F950F /* lzfse_encode_base.c in Sources */ = {isa = PBXBuildFile; fileRef = 0B5782351B8BC151002F950F /* lzfse_encode_base.c */; };
0B5782401B8BC151002F950F /* lzfse_encode.c in Sources */ = {isa = PBXBuildFile; fileRef = 0B5782371B8BC151002F950F /* lzfse_encode.c */; };
0B5782431B8BC151002F950F /* lzfse.h in Headers */ = {isa = PBXBuildFile; fileRef = 0B57823A1B8BC151002F950F /* lzfse.h */; settings = {ATTRIBUTES = (Public, ); }; };
0BB39B9F1BE140C3004F4733 /* lzfse_fse.c in Sources */ = {isa = PBXBuildFile; fileRef = 0BB39B9E1BE13F12004F4733 /* lzfse_fse.c */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
0B1AF7DD1B8D0F16005BF60D /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = 0B7CB3821B8BBEA500730478 /* Project object */;
proxyType = 1;
remoteGlobalIDString = 0B57822B1B8BC12F002F950F;
remoteInfo = "lzfse lib";
};
0B1AF7DF1B8D0F16005BF60D /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = 0B7CB3821B8BBEA500730478 /* Project object */;
proxyType = 1;
remoteGlobalIDString = 0B1AF7D11B8D0EBF005BF60D;
remoteInfo = "lzfse command";
};
0BC009291C481C600068DBA1 /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = 0B7CB3821B8BBEA500730478 /* Project object */;
proxyType = 1;
remoteGlobalIDString = 0B57822B1B8BC12F002F950F;
remoteInfo = "lzfse lib";
};
/* End PBXContainerItemProxy section */
/* Begin PBXFileReference section */
0B1AF7B71B8CF8A2005BF60D /* lzvn_decode_base.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lzvn_decode_base.c; path = src/lzvn_decode_base.c; sourceTree = "<group>"; };
0B1AF7B81B8CF8A2005BF60D /* lzvn_decode_base.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lzvn_decode_base.h; path = src/lzvn_decode_base.h; sourceTree = "<group>"; };
0B1AF7BB1B8CF8A2005BF60D /* lzvn_encode_base.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lzvn_encode_base.c; path = src/lzvn_encode_base.c; sourceTree = "<group>"; };
0B1AF7BC1B8CF8A2005BF60D /* lzvn_encode_base.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lzvn_encode_base.h; path = src/lzvn_encode_base.h; sourceTree = "<group>"; };
0B1AF7CB1B8CFF58005BF60D /* Makefile */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.make; path = Makefile; sourceTree = "<group>"; };
0B1AF7D21B8D0EBF005BF60D /* lzfse */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = lzfse; sourceTree = BUILT_PRODUCTS_DIR; };
0B1AF7E11B8D0F59005BF60D /* lzfse_main.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lzfse_main.c; path = src/lzfse_main.c; sourceTree = "<group>"; };
0B494D921D18A060002BE277 /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = "<group>"; };
0B57822C1B8BC12F002F950F /* liblzfse.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = liblzfse.a; sourceTree = BUILT_PRODUCTS_DIR; };
0B5782321B8BC151002F950F /* lzfse_decode_base.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lzfse_decode_base.c; path = src/lzfse_decode_base.c; sourceTree = "<group>"; };
0B5782341B8BC151002F950F /* lzfse_decode.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lzfse_decode.c; path = src/lzfse_decode.c; sourceTree = "<group>"; };
0B5782351B8BC151002F950F /* lzfse_encode_base.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lzfse_encode_base.c; path = src/lzfse_encode_base.c; sourceTree = "<group>"; };
0B5782361B8BC151002F950F /* lzfse_encode_tables.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lzfse_encode_tables.h; path = src/lzfse_encode_tables.h; sourceTree = "<group>"; };
0B5782371B8BC151002F950F /* lzfse_encode.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lzfse_encode.c; path = src/lzfse_encode.c; sourceTree = "<group>"; };
0B5782381B8BC151002F950F /* lzfse_fse.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lzfse_fse.h; path = src/lzfse_fse.h; sourceTree = "<group>"; };
0B5782391B8BC151002F950F /* lzfse_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lzfse_internal.h; path = src/lzfse_internal.h; sourceTree = "<group>"; };
0B57823A1B8BC151002F950F /* lzfse.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lzfse.h; path = src/lzfse.h; sourceTree = "<group>"; };
0BA652BD1D0733B20019BFE3 /* LICENSE */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = LICENSE; sourceTree = "<group>"; };
0BB39B9E1BE13F12004F4733 /* lzfse_fse.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lzfse_fse.c; path = src/lzfse_fse.c; sourceTree = "<group>"; };
6301162F1BFE759100D6CAA3 /* lzfse_tunables.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = lzfse_tunables.h; path = src/lzfse_tunables.h; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
0B1AF7CF1B8D0EBF005BF60D /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
0B1AF7E41B8D1EFD005BF60D /* liblzfse.a in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
0B5782291B8BC12F002F950F /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXFrameworksBuildPhase section */
/* Begin PBXGroup section */
0B1AF7CD1B8D0E88005BF60D /* LZFSE command */ = {
isa = PBXGroup;
children = (
0B1AF7E11B8D0F59005BF60D /* lzfse_main.c */,
);
name = "LZFSE command";
sourceTree = "<group>";
};
0B57822D1B8BC12F002F950F /* Products */ = {
isa = PBXGroup;
children = (
0B57822C1B8BC12F002F950F /* liblzfse.a */,
0B1AF7D21B8D0EBF005BF60D /* lzfse */,
);
name = Products;
sourceTree = "<group>";
};
0B5782311B8BC13D002F950F /* LZFSE lib */ = {
isa = PBXGroup;
children = (
0B57823A1B8BC151002F950F /* lzfse.h */,
6301162F1BFE759100D6CAA3 /* lzfse_tunables.h */,
0B5782391B8BC151002F950F /* lzfse_internal.h */,
0B5782381B8BC151002F950F /* lzfse_fse.h */,
0BB39B9E1BE13F12004F4733 /* lzfse_fse.c */,
0B5782361B8BC151002F950F /* lzfse_encode_tables.h */,
0B5782351B8BC151002F950F /* lzfse_encode_base.c */,
0B5782371B8BC151002F950F /* lzfse_encode.c */,
0B5782321B8BC151002F950F /* lzfse_decode_base.c */,
0B5782341B8BC151002F950F /* lzfse_decode.c */,
0B1AF7BC1B8CF8A2005BF60D /* lzvn_encode_base.h */,
0B1AF7BB1B8CF8A2005BF60D /* lzvn_encode_base.c */,
0B1AF7B81B8CF8A2005BF60D /* lzvn_decode_base.h */,
0B1AF7B71B8CF8A2005BF60D /* lzvn_decode_base.c */,
);
name = "LZFSE lib";
sourceTree = "<group>";
};
0B7CB3811B8BBEA500730478 = {
isa = PBXGroup;
children = (
0B494D921D18A060002BE277 /* README.md */,
0BA652BD1D0733B20019BFE3 /* LICENSE */,
0B1AF7CB1B8CFF58005BF60D /* Makefile */,
0B1AF7CD1B8D0E88005BF60D /* LZFSE command */,
0B5782311B8BC13D002F950F /* LZFSE lib */,
0B57822D1B8BC12F002F950F /* Products */,
);
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXHeadersBuildPhase section */
0B57822A1B8BC12F002F950F /* Headers */ = {
isa = PBXHeadersBuildPhase;
buildActionMask = 2147483647;
files = (
0B5782431B8BC151002F950F /* lzfse.h in Headers */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXHeadersBuildPhase section */
/* Begin PBXNativeTarget section */
0B1AF7D11B8D0EBF005BF60D /* lzfse command */ = {
isa = PBXNativeTarget;
buildConfigurationList = 0B1AF7D61B8D0EBF005BF60D /* Build configuration list for PBXNativeTarget "lzfse command" */;
buildPhases = (
0B1AF7CE1B8D0EBF005BF60D /* Sources */,
0B1AF7CF1B8D0EBF005BF60D /* Frameworks */,
);
buildRules = (
);
dependencies = (
0BC0092A1C481C600068DBA1 /* PBXTargetDependency */,
);
name = "lzfse command";
productName = "lzfse command";
productReference = 0B1AF7D21B8D0EBF005BF60D /* lzfse */;
productType = "com.apple.product-type.tool";
};
0B57822B1B8BC12F002F950F /* lzfse lib */ = {
isa = PBXNativeTarget;
buildConfigurationList = 0B5782301B8BC12F002F950F /* Build configuration list for PBXNativeTarget "lzfse lib" */;
buildPhases = (
0B5782281B8BC12F002F950F /* Sources */,
0B5782291B8BC12F002F950F /* Frameworks */,
0B57822A1B8BC12F002F950F /* Headers */,
);
buildRules = (
);
dependencies = (
);
name = "lzfse lib";
productName = lzfse;
productReference = 0B57822C1B8BC12F002F950F /* liblzfse.a */;
productType = "com.apple.product-type.library.static";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
0B7CB3821B8BBEA500730478 /* Project object */ = {
isa = PBXProject;
attributes = {
LastUpgradeCheck = 0700;
TargetAttributes = {
0B1AF7D11B8D0EBF005BF60D = {
CreatedOnToolsVersion = 7.0;
};
0B1AF7D91B8D0F0B005BF60D = {
CreatedOnToolsVersion = 7.0;
};
0B57822B1B8BC12F002F950F = {
CreatedOnToolsVersion = 7.0;
};
};
};
buildConfigurationList = 0B7CB3851B8BBEA500730478 /* Build configuration list for PBXProject "lzfse" */;
compatibilityVersion = "Xcode 3.2";
developmentRegion = English;
hasScannedForEncodings = 0;
knownRegions = (
en,
);
mainGroup = 0B7CB3811B8BBEA500730478;
productRefGroup = 0B57822D1B8BC12F002F950F /* Products */;
projectDirPath = "";
projectRoot = "";
targets = (
0B1AF7D91B8D0F0B005BF60D /* all */,
0B57822B1B8BC12F002F950F /* lzfse lib */,
0B1AF7D11B8D0EBF005BF60D /* lzfse command */,
);
};
/* End PBXProject section */
/* Begin PBXSourcesBuildPhase section */
0B1AF7CE1B8D0EBF005BF60D /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
0B1AF7E31B8D0F65005BF60D /* lzfse_main.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
0B5782281B8BC12F002F950F /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
0B5782401B8BC151002F950F /* lzfse_encode.c in Sources */,
0B1AF7C01B8CF8A2005BF60D /* lzvn_decode_base.c in Sources */,
0B57823D1B8BC151002F950F /* lzfse_decode.c in Sources */,
0B57823E1B8BC151002F950F /* lzfse_encode_base.c in Sources */,
0B1AF7C41B8CF8A2005BF60D /* lzvn_encode_base.c in Sources */,
0B57823B1B8BC151002F950F /* lzfse_decode_base.c in Sources */,
0BB39B9F1BE140C3004F4733 /* lzfse_fse.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin PBXTargetDependency section */
0B1AF7DE1B8D0F16005BF60D /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = 0B57822B1B8BC12F002F950F /* lzfse lib */;
targetProxy = 0B1AF7DD1B8D0F16005BF60D /* PBXContainerItemProxy */;
};
0B1AF7E01B8D0F16005BF60D /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = 0B1AF7D11B8D0EBF005BF60D /* lzfse command */;
targetProxy = 0B1AF7DF1B8D0F16005BF60D /* PBXContainerItemProxy */;
};
0BC0092A1C481C600068DBA1 /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = 0B57822B1B8BC12F002F950F /* lzfse lib */;
targetProxy = 0BC009291C481C600068DBA1 /* PBXContainerItemProxy */;
};
/* End PBXTargetDependency section */
/* Begin XCBuildConfiguration section */
0B1AF7D71B8D0EBF005BF60D /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
GCC_C_LANGUAGE_STANDARD = gnu99;
GCC_DYNAMIC_NO_PIC = NO;
GCC_NO_COMMON_BLOCKS = YES;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
"$(inherited)",
);
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
MACOSX_DEPLOYMENT_TARGET = 10.11;
MTL_ENABLE_DEBUG_INFO = YES;
ONLY_ACTIVE_ARCH = YES;
PRODUCT_NAME = lzfse;
SDKROOT = macosx;
};
name = Debug;
};
0B1AF7D81B8D0EBF005BF60D /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
GCC_C_LANGUAGE_STANDARD = gnu99;
GCC_NO_COMMON_BLOCKS = YES;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
MACOSX_DEPLOYMENT_TARGET = 10.11;
MTL_ENABLE_DEBUG_INFO = NO;
PRODUCT_NAME = lzfse;
SDKROOT = macosx;
};
name = Release;
};
0B1AF7DB1B8D0F0B005BF60D /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Debug;
};
0B1AF7DC1B8D0F0B005BF60D /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
PRODUCT_NAME = "$(TARGET_NAME)";
};
name = Release;
};
0B57822E1B8BC12F002F950F /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
EXECUTABLE_PREFIX = lib;
GCC_C_LANGUAGE_STANDARD = gnu99;
GCC_DYNAMIC_NO_PIC = NO;
GCC_NO_COMMON_BLOCKS = YES;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
"$(inherited)",
);
GCC_SYMBOLS_PRIVATE_EXTERN = YES;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
GENERATE_MASTER_OBJECT_FILE = YES;
MACOSX_DEPLOYMENT_TARGET = 10.11;
MTL_ENABLE_DEBUG_INFO = YES;
ONLY_ACTIVE_ARCH = YES;
PRODUCT_NAME = lzfse;
SDKROOT = macosx;
};
name = Debug;
};
0B57822F1B8BC12F002F950F /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
EXECUTABLE_PREFIX = lib;
GCC_C_LANGUAGE_STANDARD = gnu99;
GCC_NO_COMMON_BLOCKS = YES;
GCC_SYMBOLS_PRIVATE_EXTERN = YES;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
GENERATE_MASTER_OBJECT_FILE = YES;
MACOSX_DEPLOYMENT_TARGET = 10.11;
MTL_ENABLE_DEBUG_INFO = NO;
PRODUCT_NAME = lzfse;
SDKROOT = macosx;
};
name = Release;
};
0B7CB3861B8BBEA500730478 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
GCC_C_LANGUAGE_STANDARD = c99;
};
name = Debug;
};
0B7CB3871B8BBEA500730478 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
GCC_C_LANGUAGE_STANDARD = c99;
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
0B1AF7D61B8D0EBF005BF60D /* Build configuration list for PBXNativeTarget "lzfse command" */ = {
isa = XCConfigurationList;
buildConfigurations = (
0B1AF7D71B8D0EBF005BF60D /* Debug */,
0B1AF7D81B8D0EBF005BF60D /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
0B1AF7DA1B8D0F0B005BF60D /* Build configuration list for PBXAggregateTarget "all" */ = {
isa = XCConfigurationList;
buildConfigurations = (
0B1AF7DB1B8D0F0B005BF60D /* Debug */,
0B1AF7DC1B8D0F0B005BF60D /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
0B5782301B8BC12F002F950F /* Build configuration list for PBXNativeTarget "lzfse lib" */ = {
isa = XCConfigurationList;
buildConfigurations = (
0B57822E1B8BC12F002F950F /* Debug */,
0B57822F1B8BC12F002F950F /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
0B7CB3851B8BBEA500730478 /* Build configuration list for PBXProject "lzfse" */ = {
isa = XCConfigurationList;
buildConfigurations = (
0B7CB3861B8BBEA500730478 /* Debug */,
0B7CB3871B8BBEA500730478 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
/* End XCConfigurationList section */
};
rootObject = 0B7CB3821B8BBEA500730478 /* Project object */;
}
+136
查看文件
@@ -0,0 +1,136 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LZFSE_H
#define LZFSE_H
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#if defined(_MSC_VER) && !defined(__clang__)
# define __attribute__(X)
# pragma warning(disable : 4068)
#endif
#if defined(LZFSE_DLL)
# if defined(_WIN32) || defined(__CYGWIN__)
# if defined(LZFSE_DLL_EXPORTS)
# define LZFSE_API __declspec(dllexport)
# else
# define LZFSE_API __declspec(dllimport)
# endif
# endif
#endif
#if !defined(LZFSE_API)
# if __GNUC__ >= 4
# define LZFSE_API __attribute__((visibility("default")))
# else
# define LZFSE_API
# endif
#endif
/*! @abstract Get the required scratch buffer size to compress using LZFSE. */
LZFSE_API size_t lzfse_encode_scratch_size();
/*! @abstract Compress a buffer using LZFSE.
*
* @param dst_buffer
* Pointer to the first byte of the destination buffer.
*
* @param dst_size
* Size of the destination buffer in bytes.
*
* @param src_buffer
* Pointer to the first byte of the source buffer.
*
* @param src_size
* Size of the source buffer in bytes.
*
* @param scratch_buffer
* If non-NULL, a pointer to scratch space for the routine to use as workspace;
* the routine may use up to lzfse_encode_scratch_size( ) bytes of workspace
* during its operation, and will not perform any internal allocations. If
* NULL, the routine may allocate its own memory to use during operation via
* a single call to malloc( ), and will release it by calling free( ) prior
* to returning. For most use, passing NULL is perfectly satisfactory, but if
* you require strict control over allocation, you will want to pass an
* explicit scratch buffer.
*
* @return
* The number of bytes written to the destination buffer if the input is
* successfully compressed. If the input cannot be compressed to fit into
* the provided buffer, or an error occurs, zero is returned, and the
* contents of dst_buffer are unspecified. */
LZFSE_API size_t lzfse_encode_buffer(uint8_t *__restrict dst_buffer,
size_t dst_size,
const uint8_t *__restrict src_buffer,
size_t src_size,
void *__restrict scratch_buffer);
/*! @abstract Get the required scratch buffer size to decompress using LZFSE. */
LZFSE_API size_t lzfse_decode_scratch_size();
/*! @abstract Decompress a buffer using LZFSE.
*
* @param dst_buffer
* Pointer to the first byte of the destination buffer.
*
* @param dst_size
* Size of the destination buffer in bytes.
*
* @param src_buffer
* Pointer to the first byte of the source buffer.
*
* @param src_size
* Size of the source buffer in bytes.
*
* @param scratch_buffer
* If non-NULL, a pointer to scratch space for the routine to use as workspace;
* the routine may use up to lzfse_decode_scratch_size( ) bytes of workspace
* during its operation, and will not perform any internal allocations. If
* NULL, the routine may allocate its own memory to use during operation via
* a single call to malloc( ), and will release it by calling free( ) prior
* to returning. For most use, passing NULL is perfectly satisfactory, but if
* you require strict control over allocation, you will want to pass an
* explicit scratch buffer.
*
* @return
* The number of bytes written to the destination buffer if the input is
* successfully decompressed. If there is not enough space in the destination
* buffer to hold the entire expanded output, only the first dst_size bytes
* will be written to the buffer and dst_size is returned. Note that this
* behavior differs from that of lzfse_encode_buffer. */
LZFSE_API size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer,
size_t dst_size,
const uint8_t *__restrict src_buffer,
size_t src_size,
void *__restrict scratch_buffer);
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif /* LZFSE_H */
+72
查看文件
@@ -0,0 +1,72 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// LZFSE decode API
#include "lzfse.h"
#include "lzfse_internal.h"
size_t lzfse_decode_scratch_size() { return sizeof(lzfse_decoder_state); }
size_t lzfse_decode_buffer_with_scratch(uint8_t *__restrict dst_buffer,
size_t dst_size, const uint8_t *__restrict src_buffer,
size_t src_size, void *__restrict scratch_buffer) {
lzfse_decoder_state *s = (lzfse_decoder_state *)scratch_buffer;
memset(s, 0x00, sizeof(*s));
// Initialize state
s->src = src_buffer;
s->src_begin = src_buffer;
s->src_end = s->src + src_size;
s->dst = dst_buffer;
s->dst_begin = dst_buffer;
s->dst_end = dst_buffer + dst_size;
// Decode
int status = lzfse_decode(s);
if (status == LZFSE_STATUS_DST_FULL)
return dst_size;
if (status != LZFSE_STATUS_OK)
return 0; // failed
return (size_t)(s->dst - dst_buffer); // bytes written
}
size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size,
const uint8_t *__restrict src_buffer,
size_t src_size, void *__restrict scratch_buffer) {
int has_malloc = 0;
size_t ret = 0;
// Deal with the possible NULL pointer
if (scratch_buffer == NULL) {
// +1 in case scratch size could be zero
scratch_buffer = malloc(lzfse_decode_scratch_size() + 1);
has_malloc = 1;
}
if (scratch_buffer == NULL)
return 0;
ret = lzfse_decode_buffer_with_scratch(dst_buffer,
dst_size, src_buffer,
src_size, scratch_buffer);
if (has_malloc)
free(scratch_buffer);
return ret;
}
+630
查看文件
@@ -0,0 +1,630 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "lzfse_internal.h"
#include "lzvn_decode_base.h"
/*! @abstract Decode an entry value from next bits of stream.
* Return \p value, and set \p *nbits to the number of bits to consume
* (starting with LSB). */
static inline int lzfse_decode_v1_freq_value(uint32_t bits, int *nbits) {
static const int8_t lzfse_freq_nbits_table[32] = {
2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14,
2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14};
static const int8_t lzfse_freq_value_table[32] = {
0, 2, 1, 4, 0, 3, 1, -1, 0, 2, 1, 5, 0, 3, 1, -1,
0, 2, 1, 6, 0, 3, 1, -1, 0, 2, 1, 7, 0, 3, 1, -1};
uint32_t b = bits & 31; // lower 5 bits
int n = lzfse_freq_nbits_table[b];
*nbits = n;
// Special cases for > 5 bits encoding
if (n == 8)
return 8 + ((bits >> 4) & 0xf);
if (n == 14)
return 24 + ((bits >> 4) & 0x3ff);
// <= 5 bits encoding from table
return lzfse_freq_value_table[b];
}
/*! @abstract Extracts up to 32 bits from a 64-bit field beginning at
* \p offset, and zero-extends them to a \p uint32_t.
*
* If we number the bits of \p v from 0 (least significant) to 63 (most
* significant), the result is bits \p offset to \p offset+nbits-1. */
static inline uint32_t get_field(uint64_t v, int offset, int nbits) {
assert(offset + nbits < 64 && offset >= 0 && nbits <= 32);
if (nbits == 32)
return (uint32_t)(v >> offset);
return (uint32_t)((v >> offset) & ((1 << nbits) - 1));
}
/*! @abstract Return \c header_size field from a \c lzfse_compressed_block_header_v2. */
static inline uint32_t
lzfse_decode_v2_header_size(const lzfse_compressed_block_header_v2 *in) {
return get_field(in->packed_fields[2], 0, 32);
}
/*! @abstract Decode all fields from a \c lzfse_compressed_block_header_v2 to a
* \c lzfse_compressed_block_header_v1.
* @return 0 on success.
* @return -1 on failure. */
static inline int lzfse_decode_v1(lzfse_compressed_block_header_v1 *out,
const lzfse_compressed_block_header_v2 *in) {
// Clear all fields
memset(out, 0x00, sizeof(lzfse_compressed_block_header_v1));
uint64_t v0 = in->packed_fields[0];
uint64_t v1 = in->packed_fields[1];
uint64_t v2 = in->packed_fields[2];
out->magic = LZFSE_COMPRESSEDV1_BLOCK_MAGIC;
out->n_raw_bytes = in->n_raw_bytes;
// Literal state
out->n_literals = get_field(v0, 0, 20);
out->n_literal_payload_bytes = get_field(v0, 20, 20);
out->literal_bits = (int)get_field(v0, 60, 3) - 7;
out->literal_state[0] = get_field(v1, 0, 10);
out->literal_state[1] = get_field(v1, 10, 10);
out->literal_state[2] = get_field(v1, 20, 10);
out->literal_state[3] = get_field(v1, 30, 10);
// L,M,D state
out->n_matches = get_field(v0, 40, 20);
out->n_lmd_payload_bytes = get_field(v1, 40, 20);
out->lmd_bits = (int)get_field(v1, 60, 3) - 7;
out->l_state = get_field(v2, 32, 10);
out->m_state = get_field(v2, 42, 10);
out->d_state = get_field(v2, 52, 10);
// Total payload size
out->n_payload_bytes =
out->n_literal_payload_bytes + out->n_lmd_payload_bytes;
// Freq tables
uint16_t *dst = &(out->l_freq[0]);
const uint8_t *src = &(in->freq[0]);
const uint8_t *src_end =
(const uint8_t *)in + get_field(v2, 0, 32); // first byte after header
uint32_t accum = 0;
int accum_nbits = 0;
// No freq tables?
if (src_end == src)
return 0; // OK, freq tables were omitted
for (int i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS;
i++) {
// Refill accum, one byte at a time, until we reach end of header, or accum
// is full
while (src < src_end && accum_nbits + 8 <= 32) {
accum |= (uint32_t)(*src) << accum_nbits;
accum_nbits += 8;
src++;
}
// Decode and store value
int nbits = 0;
dst[i] = lzfse_decode_v1_freq_value(accum, &nbits);
if (nbits > accum_nbits)
return -1; // failed
// Consume nbits bits
accum >>= nbits;
accum_nbits -= nbits;
}
if (accum_nbits >= 8 || src != src_end)
return -1; // we need to end up exactly at the end of header, with less than
// 8 bits in accumulator
return 0;
}
static inline void copy(uint8_t *dst, const uint8_t *src, size_t length) {
const uint8_t *dst_end = dst + length;
do {
copy8(dst, src);
dst += 8;
src += 8;
} while (dst < dst_end);
}
static int lzfse_decode_lmd(lzfse_decoder_state *s) {
lzfse_compressed_block_decoder_state *bs = &(s->compressed_lzfse_block_state);
fse_state l_state = bs->l_state;
fse_state m_state = bs->m_state;
fse_state d_state = bs->d_state;
fse_in_stream in = bs->lmd_in_stream;
const uint8_t *src_start = s->src_begin;
const uint8_t *src = s->src + bs->lmd_in_buf;
const uint8_t *lit = bs->current_literal;
uint8_t *dst = s->dst;
uint32_t symbols = bs->n_matches;
int32_t L = bs->l_value;
int32_t M = bs->m_value;
int32_t D = bs->d_value;
assert(l_state < LZFSE_ENCODE_L_STATES);
assert(m_state < LZFSE_ENCODE_M_STATES);
assert(d_state < LZFSE_ENCODE_D_STATES);
// Number of bytes remaining in the destination buffer, minus 32 to
// provide a margin of safety for using overlarge copies on the fast path.
// This is a signed quantity, and may go negative when we are close to the
// end of the buffer. That's OK; we're careful about how we handle it
// in the slow-and-careful match execution path.
ptrdiff_t remaining_bytes = s->dst_end - dst - 32;
// If L or M is non-zero, that means that we have already started decoding
// this block, and that we needed to interrupt decoding to get more space
// from the caller. There's a pending L, M, D triplet that we weren't
// able to completely process. Jump ahead to finish executing that symbol
// before decoding new values.
if (L || M)
goto ExecuteMatch;
while (symbols > 0) {
int res;
// Decode the next L, M, D symbol from the input stream.
res = fse_in_flush(&in, &src, src_start);
if (res) {
return LZFSE_STATUS_ERROR;
}
L = fse_value_decode(&l_state, bs->l_decoder, &in);
assert(l_state < LZFSE_ENCODE_L_STATES);
if ((lit + L) >= (bs->literals + LZFSE_LITERALS_PER_BLOCK + 64)) {
return LZFSE_STATUS_ERROR;
}
res = fse_in_flush2(&in, &src, src_start);
if (res) {
return LZFSE_STATUS_ERROR;
}
M = fse_value_decode(&m_state, bs->m_decoder, &in);
assert(m_state < LZFSE_ENCODE_M_STATES);
res = fse_in_flush2(&in, &src, src_start);
if (res) {
return LZFSE_STATUS_ERROR;
}
int32_t new_d = fse_value_decode(&d_state, bs->d_decoder, &in);
assert(d_state < LZFSE_ENCODE_D_STATES);
D = new_d ? new_d : D;
symbols--;
ExecuteMatch:
// Error if D is out of range, so that we avoid passing through
// uninitialized data or accesssing memory out of the destination
// buffer.
if ((uint32_t)D > dst + L - s->dst_begin)
return LZFSE_STATUS_ERROR;
if (L + M <= remaining_bytes) {
// If we have plenty of space remaining, we can copy the literal
// and match with 16- and 32-byte operations, without worrying
// about writing off the end of the buffer.
remaining_bytes -= L + M;
copy(dst, lit, L);
dst += L;
lit += L;
// For the match, we have two paths; a fast copy by 16-bytes if
// the match distance is large enough to allow it, and a more
// careful path that applies a permutation to account for the
// possible overlap between source and destination if the distance
// is small.
if (D >= 8 || D >= M)
copy(dst, dst - D, M);
else
for (size_t i = 0; i < M; i++)
dst[i] = dst[i - D];
dst += M;
}
else {
// Otherwise, we are very close to the end of the destination
// buffer, so we cannot use wide copies that slop off the end
// of the region that we are copying to. First, we restore
// the true length remaining, rather than the sham value we've
// been using so far.
remaining_bytes += 32;
// Now, we process the literal. Either there's space for it
// or there isn't; if there is, we copy the whole thing and
// update all the pointers and lengths to reflect the copy.
if (L <= remaining_bytes) {
for (size_t i = 0; i < L; i++)
dst[i] = lit[i];
dst += L;
lit += L;
remaining_bytes -= L;
L = 0;
}
// There isn't enough space to fit the whole literal. Copy as
// much of it as we can, update the pointers and the value of
// L, and report that the destination buffer is full. Note that
// we always write right up to the end of the destination buffer.
else {
for (size_t i = 0; i < remaining_bytes; i++)
dst[i] = lit[i];
dst += remaining_bytes;
lit += remaining_bytes;
L -= remaining_bytes;
goto DestinationBufferIsFull;
}
// The match goes just like the literal does. We copy as much as
// we can byte-by-byte, and if we reach the end of the buffer
// before finishing, we return to the caller indicating that
// the buffer is full.
if (M <= remaining_bytes) {
for (size_t i = 0; i < M; i++)
dst[i] = dst[i - D];
dst += M;
remaining_bytes -= M;
M = 0;
(void)M; // no dead store warning
// We don't need to update M = 0, because there's no partial
// symbol to continue executing. Either we're at the end of
// the block, in which case we will never need to resume with
// this state, or we're going to decode another L, M, D set,
// which will overwrite M anyway.
//
// But we still set M = 0, to maintain the post-condition.
} else {
for (size_t i = 0; i < remaining_bytes; i++)
dst[i] = dst[i - D];
dst += remaining_bytes;
M -= remaining_bytes;
DestinationBufferIsFull:
// Because we want to be able to resume decoding where we've left
// off (even in the middle of a literal or match), we need to
// update all of the block state fields with the current values
// so that we can resume execution from this point once the
// caller has given us more space to write into.
bs->l_value = L;
bs->m_value = M;
bs->d_value = D;
bs->l_state = l_state;
bs->m_state = m_state;
bs->d_state = d_state;
bs->lmd_in_stream = in;
bs->n_matches = symbols;
bs->lmd_in_buf = (uint32_t)(src - s->src);
bs->current_literal = lit;
s->dst = dst;
return LZFSE_STATUS_DST_FULL;
}
// Restore the "sham" decremented value of remaining_bytes and
// continue to the next L, M, D triple. We'll just be back in
// the careful path again, but this only happens at the very end
// of the buffer, so a little minor inefficiency here is a good
// tradeoff for simpler code.
remaining_bytes -= 32;
}
}
// Because we've finished with the whole block, we don't need to update
// any of the blockstate fields; they will not be used again. We just
// update the destination pointer in the state object and return.
s->dst = dst;
return LZFSE_STATUS_OK;
}
int lzfse_decode(lzfse_decoder_state *s) {
while (1) {
// Are we inside a block?
switch (s->block_magic) {
case LZFSE_NO_BLOCK_MAGIC: {
// We need at least 4 bytes of magic number to identify next block
if (s->src + 4 > s->src_end)
return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
uint32_t magic = load4(s->src);
if (magic == LZFSE_ENDOFSTREAM_BLOCK_MAGIC) {
s->src += 4;
s->end_of_stream = 1;
return LZFSE_STATUS_OK; // done
}
if (magic == LZFSE_UNCOMPRESSED_BLOCK_MAGIC) {
if (s->src + sizeof(uncompressed_block_header) > s->src_end)
return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
// Setup state for uncompressed block
uncompressed_block_decoder_state *bs = &(s->uncompressed_block_state);
bs->n_raw_bytes =
load4(s->src + offsetof(uncompressed_block_header, n_raw_bytes));
s->src += sizeof(uncompressed_block_header);
s->block_magic = magic;
break;
}
if (magic == LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC) {
if (s->src + sizeof(lzvn_compressed_block_header) > s->src_end)
return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
// Setup state for compressed LZVN block
lzvn_compressed_block_decoder_state *bs =
&(s->compressed_lzvn_block_state);
bs->n_raw_bytes =
load4(s->src + offsetof(lzvn_compressed_block_header, n_raw_bytes));
bs->n_payload_bytes = load4(
s->src + offsetof(lzvn_compressed_block_header, n_payload_bytes));
bs->d_prev = 0;
s->src += sizeof(lzvn_compressed_block_header);
s->block_magic = magic;
break;
}
if (magic == LZFSE_COMPRESSEDV1_BLOCK_MAGIC ||
magic == LZFSE_COMPRESSEDV2_BLOCK_MAGIC) {
lzfse_compressed_block_header_v1 header1;
size_t header_size = 0;
// Decode compressed headers
if (magic == LZFSE_COMPRESSEDV2_BLOCK_MAGIC) {
// Check we have the fixed part of the structure
if (s->src + offsetof(lzfse_compressed_block_header_v2, freq) > s->src_end)
return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
// Get size, and check we have the entire structure
const lzfse_compressed_block_header_v2 *header2 =
(const lzfse_compressed_block_header_v2 *)s->src; // not aligned, OK
header_size = lzfse_decode_v2_header_size(header2);
if (s->src + header_size > s->src_end)
return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
int decodeStatus = lzfse_decode_v1(&header1, header2);
if (decodeStatus != 0)
return LZFSE_STATUS_ERROR; // failed
} else {
if (s->src + sizeof(lzfse_compressed_block_header_v1) > s->src_end)
return LZFSE_STATUS_SRC_EMPTY; // SRC truncated
memcpy(&header1, s->src, sizeof(lzfse_compressed_block_header_v1));
header_size = sizeof(lzfse_compressed_block_header_v1);
}
// We require the header + entire encoded block to be present in SRC
// during the entire block decoding.
// This can be relaxed somehow, if it becomes a limiting factor, at the
// price of a more complex state maintenance.
// For DST, we can't easily require space for the entire decoded block,
// because it may expand to something very very large.
if (s->src + header_size + header1.n_literal_payload_bytes +
header1.n_lmd_payload_bytes >
s->src_end)
return LZFSE_STATUS_SRC_EMPTY; // need all encoded block
// Sanity checks
if (lzfse_check_block_header_v1(&header1) != 0) {
return LZFSE_STATUS_ERROR;
}
// Skip header
s->src += header_size;
// Setup state for compressed V1 block from header
lzfse_compressed_block_decoder_state *bs =
&(s->compressed_lzfse_block_state);
bs->n_lmd_payload_bytes = header1.n_lmd_payload_bytes;
bs->n_matches = header1.n_matches;
fse_init_decoder_table(LZFSE_ENCODE_LITERAL_STATES,
LZFSE_ENCODE_LITERAL_SYMBOLS,
header1.literal_freq, bs->literal_decoder);
fse_init_value_decoder_table(
LZFSE_ENCODE_L_STATES, LZFSE_ENCODE_L_SYMBOLS, header1.l_freq,
l_extra_bits, l_base_value, bs->l_decoder);
fse_init_value_decoder_table(
LZFSE_ENCODE_M_STATES, LZFSE_ENCODE_M_SYMBOLS, header1.m_freq,
m_extra_bits, m_base_value, bs->m_decoder);
fse_init_value_decoder_table(
LZFSE_ENCODE_D_STATES, LZFSE_ENCODE_D_SYMBOLS, header1.d_freq,
d_extra_bits, d_base_value, bs->d_decoder);
// Decode literals
{
fse_in_stream in;
const uint8_t *buf_start = s->src_begin;
s->src += header1.n_literal_payload_bytes; // skip literal payload
const uint8_t *buf = s->src; // read bits backwards from the end
if (fse_in_init(&in, header1.literal_bits, &buf, buf_start) != 0)
return LZFSE_STATUS_ERROR;
fse_state state0 = header1.literal_state[0];
fse_state state1 = header1.literal_state[1];
fse_state state2 = header1.literal_state[2];
fse_state state3 = header1.literal_state[3];
for (uint32_t i = 0; i < header1.n_literals; i += 4) // n_literals is multiple of 4
{
#if FSE_IOSTREAM_64
if (fse_in_flush(&in, &buf, buf_start) != 0)
return LZFSE_STATUS_ERROR; // [57, 64] bits
bs->literals[i + 0] =
fse_decode(&state0, bs->literal_decoder, &in); // 10b max
bs->literals[i + 1] =
fse_decode(&state1, bs->literal_decoder, &in); // 10b max
bs->literals[i + 2] =
fse_decode(&state2, bs->literal_decoder, &in); // 10b max
bs->literals[i + 3] =
fse_decode(&state3, bs->literal_decoder, &in); // 10b max
#else
if (fse_in_flush(&in, &buf, buf_start) != 0)
return LZFSE_STATUS_ERROR; // [25, 23] bits
bs->literals[i + 0] =
fse_decode(&state0, bs->literal_decoder, &in); // 10b max
bs->literals[i + 1] =
fse_decode(&state1, bs->literal_decoder, &in); // 10b max
if (fse_in_flush(&in, &buf, buf_start) != 0)
return LZFSE_STATUS_ERROR; // [25, 23] bits
bs->literals[i + 2] =
fse_decode(&state2, bs->literal_decoder, &in); // 10b max
bs->literals[i + 3] =
fse_decode(&state3, bs->literal_decoder, &in); // 10b max
#endif
}
bs->current_literal = bs->literals;
} // literals
// SRC is not incremented to skip the LMD payload, since we need it
// during block decode.
// We will increment SRC at the end of the block only after this point.
// Initialize the L,M,D decode stream, do not start decoding matches
// yet, and store decoder state
{
fse_in_stream in;
// read bits backwards from the end
const uint8_t *buf = s->src + header1.n_lmd_payload_bytes;
if (fse_in_init(&in, header1.lmd_bits, &buf, s->src) != 0)
return LZFSE_STATUS_ERROR;
bs->l_state = header1.l_state;
bs->m_state = header1.m_state;
bs->d_state = header1.d_state;
bs->lmd_in_buf = (uint32_t)(buf - s->src);
bs->l_value = bs->m_value = 0;
// Initialize D to an illegal value so we can't erroneously use
// an uninitialized "previous" value.
bs->d_value = -1;
bs->lmd_in_stream = in;
}
s->block_magic = magic;
break;
}
// Here we have an invalid magic number
return LZFSE_STATUS_ERROR;
} // LZFSE_NO_BLOCK_MAGIC
case LZFSE_UNCOMPRESSED_BLOCK_MAGIC: {
uncompressed_block_decoder_state *bs = &(s->uncompressed_block_state);
// Compute the size (in bytes) of the data that we will actually copy.
// This size is minimum(bs->n_raw_bytes, space in src, space in dst).
uint32_t copy_size = bs->n_raw_bytes; // bytes left to copy
if (copy_size == 0) {
s->block_magic = 0;
break;
} // end of block
if (s->src_end <= s->src)
return LZFSE_STATUS_SRC_EMPTY; // need more SRC data
const size_t src_space = s->src_end - s->src;
if (copy_size > src_space)
copy_size = (uint32_t)src_space; // limit to SRC data (> 0)
if (s->dst_end <= s->dst)
return LZFSE_STATUS_DST_FULL; // need more DST capacity
const size_t dst_space = s->dst_end - s->dst;
if (copy_size > dst_space)
copy_size = (uint32_t)dst_space; // limit to DST capacity (> 0)
// Now that we know that the copy size is bounded to the source and
// dest buffers, go ahead and copy the data.
// We always have copy_size > 0 here
memcpy(s->dst, s->src, copy_size);
s->src += copy_size;
s->dst += copy_size;
bs->n_raw_bytes -= copy_size;
break;
} // LZFSE_UNCOMPRESSED_BLOCK_MAGIC
case LZFSE_COMPRESSEDV1_BLOCK_MAGIC:
case LZFSE_COMPRESSEDV2_BLOCK_MAGIC: {
lzfse_compressed_block_decoder_state *bs =
&(s->compressed_lzfse_block_state);
// Require the entire LMD payload to be in SRC
if (s->src_end <= s->src ||
bs->n_lmd_payload_bytes > (size_t)(s->src_end - s->src))
return LZFSE_STATUS_SRC_EMPTY;
int status = lzfse_decode_lmd(s);
if (status != LZFSE_STATUS_OK)
return status;
s->block_magic = LZFSE_NO_BLOCK_MAGIC;
s->src += bs->n_lmd_payload_bytes; // to next block
break;
} // LZFSE_COMPRESSEDV1_BLOCK_MAGIC || LZFSE_COMPRESSEDV2_BLOCK_MAGIC
case LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC: {
lzvn_compressed_block_decoder_state *bs =
&(s->compressed_lzvn_block_state);
if (bs->n_payload_bytes > 0 && s->src_end <= s->src)
return LZFSE_STATUS_SRC_EMPTY; // need more SRC data
// Init LZVN decoder state
lzvn_decoder_state dstate;
memset(&dstate, 0x00, sizeof(dstate));
dstate.src = s->src;
dstate.src_end = s->src_end;
if (dstate.src_end - s->src > bs->n_payload_bytes)
dstate.src_end = s->src + bs->n_payload_bytes; // limit to payload bytes
dstate.dst_begin = s->dst_begin;
dstate.dst = s->dst;
dstate.dst_end = s->dst_end;
if (dstate.dst_end - s->dst > bs->n_raw_bytes)
dstate.dst_end = s->dst + bs->n_raw_bytes; // limit to raw bytes
dstate.d_prev = bs->d_prev;
dstate.end_of_stream = 0;
// Run LZVN decoder
lzvn_decode(&dstate);
// Update our state
size_t src_used = dstate.src - s->src;
size_t dst_used = dstate.dst - s->dst;
if (src_used > bs->n_payload_bytes || dst_used > bs->n_raw_bytes)
return LZFSE_STATUS_ERROR; // sanity check
s->src = dstate.src;
s->dst = dstate.dst;
bs->n_payload_bytes -= (uint32_t)src_used;
bs->n_raw_bytes -= (uint32_t)dst_used;
bs->d_prev = (uint32_t)dstate.d_prev;
// Test end of block
if (bs->n_payload_bytes == 0 && bs->n_raw_bytes == 0 &&
dstate.end_of_stream) {
s->block_magic = 0;
break;
} // block done
// Check for invalid state
if (bs->n_payload_bytes == 0 || bs->n_raw_bytes == 0 ||
dstate.end_of_stream)
return LZFSE_STATUS_ERROR;
// Here, block is not done and state is valid, so we need more space in dst.
return LZFSE_STATUS_DST_FULL;
}
default:
return LZFSE_STATUS_ERROR; // invalid magic
} // switch magic
} // block loop
return LZFSE_STATUS_OK;
}
+163
查看文件
@@ -0,0 +1,163 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// LZFSE encode API
#include "lzfse.h"
#include "lzfse_internal.h"
size_t lzfse_encode_scratch_size() {
size_t s1 = sizeof(lzfse_encoder_state);
size_t s2 = lzvn_encode_scratch_size();
return (s1 > s2) ? s1 : s2; // max(lzfse,lzvn)
}
size_t lzfse_encode_buffer_with_scratch(uint8_t *__restrict dst_buffer,
size_t dst_size, const uint8_t *__restrict src_buffer,
size_t src_size, void *__restrict scratch_buffer) {
const size_t original_size = src_size;
// If input is really really small, go directly to uncompressed buffer
// (because LZVN will refuse to encode it, and we will report a failure)
if (src_size < LZVN_ENCODE_MIN_SRC_SIZE)
goto try_uncompressed;
// If input is too small, try encoding with LZVN
if (src_size < LZFSE_ENCODE_LZVN_THRESHOLD) {
// need header + end-of-stream marker
size_t extra_size = 4 + sizeof(lzvn_compressed_block_header);
if (dst_size <= extra_size)
goto try_uncompressed; // DST is really too small, give up
size_t sz = lzvn_encode_buffer(
dst_buffer + sizeof(lzvn_compressed_block_header),
dst_size - extra_size, src_buffer, src_size, scratch_buffer);
if (sz == 0 || sz >= src_size)
goto try_uncompressed; // failed, or no compression, fall back to
// uncompressed block
// If we could encode, setup header and end-of-stream marker (we left room
// for them, no need to test)
lzvn_compressed_block_header header;
header.magic = LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC;
header.n_raw_bytes = (uint32_t)src_size;
header.n_payload_bytes = (uint32_t)sz;
memcpy(dst_buffer, &header, sizeof(header));
store4(dst_buffer + sizeof(lzvn_compressed_block_header) + sz,
LZFSE_ENDOFSTREAM_BLOCK_MAGIC);
return sz + extra_size;
}
// Try encoding with LZFSE
{
lzfse_encoder_state *state = scratch_buffer;
memset(state, 0x00, sizeof *state);
if (lzfse_encode_init(state) != LZFSE_STATUS_OK)
goto try_uncompressed;
state->dst = dst_buffer;
state->dst_begin = dst_buffer;
state->dst_end = &dst_buffer[dst_size];
state->src = src_buffer;
state->src_encode_i = 0;
if (src_size >= 0xffffffffU) {
// lzfse only uses 32 bits for offsets internally, so if the input
// buffer is really huge, we need to process it in smaller chunks.
// Note that we switch over to this path for sizes much smaller
// 2GB because it's actually faster to change algorithms well before
// it's necessary for correctness.
// The first chunk, we just process normally.
const lzfse_offset encoder_block_size = 262144;
state->src_end = encoder_block_size;
if (lzfse_encode_base(state) != LZFSE_STATUS_OK)
goto try_uncompressed;
src_size -= encoder_block_size;
while (src_size >= encoder_block_size) {
// All subsequent chunks require a translation to keep the offsets
// from getting too big. Note that we are always going from
// encoder_block_size up to 2*encoder_block_size so that the
// offsets remain positive (as opposed to resetting to zero and
// having negative offsets).
state->src_end = 2 * encoder_block_size;
if (lzfse_encode_base(state) != LZFSE_STATUS_OK)
goto try_uncompressed;
lzfse_encode_translate(state, encoder_block_size);
src_size -= encoder_block_size;
}
// Set the end for the final chunk.
state->src_end = encoder_block_size + (lzfse_offset)src_size;
}
// If the source buffer is small enough to use 32-bit offsets, we simply
// encode the whole thing in a single chunk.
else
state->src_end = (lzfse_offset)src_size;
// This is either the trailing chunk (if the source file is huge), or
// the whole source file.
if (lzfse_encode_base(state) != LZFSE_STATUS_OK)
goto try_uncompressed;
if (lzfse_encode_finish(state) != LZFSE_STATUS_OK)
goto try_uncompressed;
// No error occured, return compressed size.
return state->dst - dst_buffer;
}
try_uncompressed:
// Compression failed for some reason. If we can fit the data into the
// output buffer uncompressed, go ahead and do that instead.
if (original_size + 12 <= dst_size && original_size < INT32_MAX) {
uncompressed_block_header header = {.magic = LZFSE_UNCOMPRESSED_BLOCK_MAGIC,
.n_raw_bytes = (uint32_t)src_size};
uint8_t *dst_end = dst_buffer;
memcpy(dst_end, &header, sizeof header);
dst_end += sizeof header;
memcpy(dst_end, src_buffer, original_size);
dst_end += original_size;
store4(dst_end, LZFSE_ENDOFSTREAM_BLOCK_MAGIC);
dst_end += 4;
return dst_end - dst_buffer;
}
// Otherwise, there's nothing we can do, so return zero.
return 0;
}
size_t lzfse_encode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size,
const uint8_t *__restrict src_buffer,
size_t src_size, void *__restrict scratch_buffer) {
int has_malloc = 0;
size_t ret = 0;
// Deal with the possible NULL pointer
if (scratch_buffer == NULL) {
// +1 in case scratch size could be zero
scratch_buffer = malloc(lzfse_encode_scratch_size() + 1);
has_malloc = 1;
}
if (scratch_buffer == NULL)
return 0;
ret = lzfse_encode_buffer_with_scratch(dst_buffer,
dst_size, src_buffer,
src_size, scratch_buffer);
if (has_malloc)
free(scratch_buffer);
return ret;
}
+830
查看文件
@@ -0,0 +1,830 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// LZFSE encoder
#include "lzfse_internal.h"
#include "lzfse_encode_tables.h"
/*! @abstract Get hash in range [0, LZFSE_ENCODE_HASH_VALUES-1] from 4 bytes in X. */
static inline uint32_t hashX(uint32_t x) {
return (x * 2654435761U) >>
(32 - LZFSE_ENCODE_HASH_BITS); // Knuth multiplicative hash
}
/*! @abstract Return value with all 0 except nbits<=32 unsigned bits from V
* at bit offset OFFSET.
* V is assumed to fit on nbits bits. */
static inline uint64_t setField(uint32_t v, int offset, int nbits) {
assert(offset + nbits < 64 && offset >= 0 && nbits <= 32);
assert(nbits == 32 || (v < (1U << nbits)));
return ((uint64_t)v << (uint64_t)offset);
}
/*! @abstract Encode all fields, except freq, from a
* lzfse_compressed_block_header_v1 to a lzfse_compressed_block_header_v2.
* All but the header_size and freq fields of the output are modified. */
static inline void
lzfse_encode_v1_state(lzfse_compressed_block_header_v2 *out,
const lzfse_compressed_block_header_v1 *in) {
out->magic = LZFSE_COMPRESSEDV2_BLOCK_MAGIC;
out->n_raw_bytes = in->n_raw_bytes;
// Literal state
out->packed_fields[0] = setField(in->n_literals, 0, 20) |
setField(in->n_literal_payload_bytes, 20, 20) |
setField(in->n_matches, 40, 20) |
setField(7 + in->literal_bits, 60, 3);
out->packed_fields[1] = setField(in->literal_state[0], 0, 10) |
setField(in->literal_state[1], 10, 10) |
setField(in->literal_state[2], 20, 10) |
setField(in->literal_state[3], 30, 10) |
setField(in->n_lmd_payload_bytes, 40, 20) |
setField(7 + in->lmd_bits, 60, 3);
out->packed_fields[2] = out->packed_fields[2] // header_size already stored in v[2]
| setField(in->l_state, 32, 10) | setField(in->m_state, 42, 10) |
setField(in->d_state, 52, 10);
}
/*! @abstract Encode an entry value in a freq table. Return bits, and sets
* *nbits to the number of bits to serialize. */
static inline uint32_t lzfse_encode_v1_freq_value(int value, int *nbits) {
// Fixed Huffman code, bits are read from LSB.
// Note that we rely on the position of the first '0' bit providing the number
// of bits.
switch (value) {
case 0:
*nbits = 2;
return 0; // 0.0
case 1:
*nbits = 2;
return 2; // 1.0
case 2:
*nbits = 3;
return 1; // 0.01
case 3:
*nbits = 3;
return 5; // 1.01
case 4:
*nbits = 5;
return 3; // 00.011
case 5:
*nbits = 5;
return 11; // 01.011
case 6:
*nbits = 5;
return 19; // 10.011
case 7:
*nbits = 5;
return 27; // 11.011
default:
break;
}
if (value < 24) {
*nbits = 8; // 4+4
return 7 + ((value - 8) << 4); // xxxx.0111
}
// 24..1047
*nbits = 14; // 4+10
return ((value - 24) << 4) + 15; // xxxxxxxxxx.1111
}
/*! @abstract Encode all tables from a lzfse_compressed_block_header_v1
* to a lzfse_compressed_block_header_v2.
* Only the header_size and freq fields of the output are modified.
* @return Size of the lzfse_compressed_block_header_v2 */
static inline size_t
lzfse_encode_v1_freq_table(lzfse_compressed_block_header_v2 *out,
const lzfse_compressed_block_header_v1 *in) {
uint32_t accum = 0;
int accum_nbits = 0;
const uint16_t *src = &(in->l_freq[0]); // first value of first table (struct
// will not be modified, so this code
// will remain valid)
uint8_t *dst = &(out->freq[0]);
for (int i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS;
i++) {
// Encode one value to accum
int nbits = 0;
uint32_t bits = lzfse_encode_v1_freq_value(src[i], &nbits);
assert(bits < (1 << nbits));
accum |= bits << accum_nbits;
accum_nbits += nbits;
// Store bytes from accum to output buffer
while (accum_nbits >= 8) {
*dst = (uint8_t)(accum & 0xff);
accum >>= 8;
accum_nbits -= 8;
dst++;
}
}
// Store final byte if needed
if (accum_nbits > 0) {
*dst = (uint8_t)(accum & 0xff);
dst++;
}
// Return final size of out
uint32_t header_size = (uint32_t)(dst - (uint8_t *)out);
out->packed_fields[0] = 0;
out->packed_fields[1] = 0;
out->packed_fields[2] = setField(header_size, 0, 32);
return header_size;
}
// We need to limit forward match length to make sure it won't split into a too
// large number of LMD.
// The limit itself is quite large, so it doesn't really impact compression
// ratio.
// The matches may still be expanded backwards by a few bytes, so the final
// length may be greater than this limit, which is OK.
#define LZFSE_ENCODE_MAX_MATCH_LENGTH (100 * LZFSE_ENCODE_MAX_M_VALUE)
// ===============================================================
// Encoder back end
/*! @abstract Encode matches stored in STATE into a compressed/uncompressed block.
* @return LZFSE_STATUS_OK on success.
* @return LZFSE_STATUS_DST_FULL and restore initial state if output buffer is
* full. */
static int lzfse_encode_matches(lzfse_encoder_state *s) {
if (s->n_literals == 0 && s->n_matches == 0)
return LZFSE_STATUS_OK; // nothing to store, OK
uint32_t l_occ[LZFSE_ENCODE_L_SYMBOLS];
uint32_t m_occ[LZFSE_ENCODE_M_SYMBOLS];
uint32_t d_occ[LZFSE_ENCODE_D_SYMBOLS];
uint32_t literal_occ[LZFSE_ENCODE_LITERAL_SYMBOLS];
fse_encoder_entry l_encoder[LZFSE_ENCODE_L_SYMBOLS];
fse_encoder_entry m_encoder[LZFSE_ENCODE_M_SYMBOLS];
fse_encoder_entry d_encoder[LZFSE_ENCODE_D_SYMBOLS];
fse_encoder_entry literal_encoder[LZFSE_ENCODE_LITERAL_SYMBOLS];
int ok = 1;
lzfse_compressed_block_header_v1 header1 = {0};
lzfse_compressed_block_header_v2 *header2 = 0;
// Keep initial state to be able to restore it if DST full
uint8_t *dst0 = s->dst;
uint32_t n_literals0 = s->n_literals;
// Add 0x00 literals until n_literals multiple of 4, since we encode 4
// interleaved literal streams.
while (s->n_literals & 3) {
uint32_t n = s->n_literals++;
s->literals[n] = 0;
}
// Encode previous distance
uint32_t d_prev = 0;
for (uint32_t i = 0; i < s->n_matches; i++) {
uint32_t d = s->d_values[i];
if (d == d_prev)
s->d_values[i] = 0;
else
d_prev = d;
}
// Clear occurrence tables
memset(l_occ, 0, sizeof(l_occ));
memset(m_occ, 0, sizeof(m_occ));
memset(d_occ, 0, sizeof(d_occ));
memset(literal_occ, 0, sizeof(literal_occ));
// Update occurrence tables in all 4 streams (L,M,D,literals)
uint32_t l_sum = 0;
uint32_t m_sum = 0;
for (uint32_t i = 0; i < s->n_matches; i++) {
uint32_t l = s->l_values[i];
l_sum += l;
l_occ[l_base_from_value(l)]++;
}
for (uint32_t i = 0; i < s->n_matches; i++) {
uint32_t m = s->m_values[i];
m_sum += m;
m_occ[m_base_from_value(m)]++;
}
for (uint32_t i = 0; i < s->n_matches; i++)
d_occ[d_base_from_value(s->d_values[i])]++;
for (uint32_t i = 0; i < s->n_literals; i++)
literal_occ[s->literals[i]]++;
// Make sure we have enough room for a _full_ V2 header
if (s->dst + sizeof(lzfse_compressed_block_header_v2) > s->dst_end) {
ok = 0;
goto END;
}
header2 = (lzfse_compressed_block_header_v2 *)(s->dst);
// Setup header V1
header1.magic = LZFSE_COMPRESSEDV1_BLOCK_MAGIC;
header1.n_raw_bytes = m_sum + l_sum;
header1.n_matches = s->n_matches;
header1.n_literals = s->n_literals;
// Normalize occurrence tables to freq tables
fse_normalize_freq(LZFSE_ENCODE_L_STATES, LZFSE_ENCODE_L_SYMBOLS, l_occ,
header1.l_freq);
fse_normalize_freq(LZFSE_ENCODE_M_STATES, LZFSE_ENCODE_M_SYMBOLS, m_occ,
header1.m_freq);
fse_normalize_freq(LZFSE_ENCODE_D_STATES, LZFSE_ENCODE_D_SYMBOLS, d_occ,
header1.d_freq);
fse_normalize_freq(LZFSE_ENCODE_LITERAL_STATES, LZFSE_ENCODE_LITERAL_SYMBOLS,
literal_occ, header1.literal_freq);
// Compress freq tables to V2 header, and get actual size of V2 header
s->dst += lzfse_encode_v1_freq_table(header2, &header1);
// Initialize encoder tables from freq tables
fse_init_encoder_table(LZFSE_ENCODE_L_STATES, LZFSE_ENCODE_L_SYMBOLS,
header1.l_freq, l_encoder);
fse_init_encoder_table(LZFSE_ENCODE_M_STATES, LZFSE_ENCODE_M_SYMBOLS,
header1.m_freq, m_encoder);
fse_init_encoder_table(LZFSE_ENCODE_D_STATES, LZFSE_ENCODE_D_SYMBOLS,
header1.d_freq, d_encoder);
fse_init_encoder_table(LZFSE_ENCODE_LITERAL_STATES,
LZFSE_ENCODE_LITERAL_SYMBOLS, header1.literal_freq,
literal_encoder);
// Encode literals
{
fse_out_stream out;
fse_out_init(&out);
fse_state state0, state1, state2, state3;
state0 = state1 = state2 = state3 = 0;
uint8_t *buf = s->dst;
uint32_t i = s->n_literals; // I multiple of 4
// We encode starting from the last literal so we can decode starting from
// the first
while (i > 0) {
if (buf + 16 > s->dst_end) {
ok = 0;
goto END;
} // out full
i -= 4;
fse_encode(&state3, literal_encoder, &out, s->literals[i + 3]); // 10b
fse_encode(&state2, literal_encoder, &out, s->literals[i + 2]); // 10b
#if !FSE_IOSTREAM_64
fse_out_flush(&out, &buf);
#endif
fse_encode(&state1, literal_encoder, &out, s->literals[i + 1]); // 10b
fse_encode(&state0, literal_encoder, &out, s->literals[i + 0]); // 10b
fse_out_flush(&out, &buf);
}
fse_out_finish(&out, &buf);
// Update header with final encoder state
header1.literal_bits = out.accum_nbits; // [-7, 0]
header1.n_literal_payload_bytes = (uint32_t)(buf - s->dst);
header1.literal_state[0] = state0;
header1.literal_state[1] = state1;
header1.literal_state[2] = state2;
header1.literal_state[3] = state3;
// Update state
s->dst = buf;
} // literals
// Encode L,M,D
{
fse_out_stream out;
fse_out_init(&out);
fse_state l_state, m_state, d_state;
l_state = m_state = d_state = 0;
uint8_t *buf = s->dst;
uint32_t i = s->n_matches;
// Add 8 padding bytes to the L,M,D payload
if (buf + 8 > s->dst_end) {
ok = 0;
goto END;
} // out full
store8(buf, 0);
buf += 8;
// We encode starting from the last match so we can decode starting from the
// first
while (i > 0) {
if (buf + 16 > s->dst_end) {
ok = 0;
goto END;
} // out full
i -= 1;
// D requires 23b max
int32_t d_value = s->d_values[i];
uint8_t d_symbol = d_base_from_value(d_value);
int32_t d_nbits = d_extra_bits[d_symbol];
int32_t d_bits = d_value - d_base_value[d_symbol];
fse_out_push(&out, d_nbits, d_bits);
fse_encode(&d_state, d_encoder, &out, d_symbol);
#if !FSE_IOSTREAM_64
fse_out_flush(&out, &buf);
#endif
// M requires 17b max
int32_t m_value = s->m_values[i];
uint8_t m_symbol = m_base_from_value(m_value);
int32_t m_nbits = m_extra_bits[m_symbol];
int32_t m_bits = m_value - m_base_value[m_symbol];
fse_out_push(&out, m_nbits, m_bits);
fse_encode(&m_state, m_encoder, &out, m_symbol);
#if !FSE_IOSTREAM_64
fse_out_flush(&out, &buf);
#endif
// L requires 14b max
int32_t l_value = s->l_values[i];
uint8_t l_symbol = l_base_from_value(l_value);
int32_t l_nbits = l_extra_bits[l_symbol];
int32_t l_bits = l_value - l_base_value[l_symbol];
fse_out_push(&out, l_nbits, l_bits);
fse_encode(&l_state, l_encoder, &out, l_symbol);
fse_out_flush(&out, &buf);
}
fse_out_finish(&out, &buf);
// Update header with final encoder state
header1.n_lmd_payload_bytes = (uint32_t)(buf - s->dst);
header1.lmd_bits = out.accum_nbits; // [-7, 0]
header1.l_state = l_state;
header1.m_state = m_state;
header1.d_state = d_state;
// Update state
s->dst = buf;
} // L,M,D
// Final state update, here we had enough space in DST, and are not going to
// revert state
s->n_literals = 0;
s->n_matches = 0;
// Final payload size
header1.n_payload_bytes =
header1.n_literal_payload_bytes + header1.n_lmd_payload_bytes;
// Encode state info in V2 header (we previously encoded the tables, now we
// set the other fields)
lzfse_encode_v1_state(header2, &header1);
END:
if (!ok) {
// Revert state, DST was full
// Revert the d_prev encoding
uint32_t d_prev = 0;
for (uint32_t i = 0; i < s->n_matches; i++) {
uint32_t d = s->d_values[i];
if (d == 0)
s->d_values[i] = d_prev;
else
d_prev = d;
}
// Revert literal count
s->n_literals = n_literals0;
// Revert DST
s->dst = dst0;
return LZFSE_STATUS_DST_FULL; // DST full
}
return LZFSE_STATUS_OK;
}
/*! @abstract Push a L,M,D match into the STATE.
* @return LZFSE_STATUS_OK if OK.
* @return LZFSE_STATUS_DST_FULL if the match can't be pushed, meaning one of
* the buffers is full. In that case the state is not modified. */
static inline int lzfse_push_lmd(lzfse_encoder_state *s, uint32_t L,
uint32_t M, uint32_t D) {
// Check if we have enough space to push the match (we add some margin to copy
// literals faster here, and round final count later)
if (s->n_matches + 1 + 8 > LZFSE_MATCHES_PER_BLOCK)
return LZFSE_STATUS_DST_FULL; // state full
if (s->n_literals + L + 16 > LZFSE_LITERALS_PER_BLOCK)
return LZFSE_STATUS_DST_FULL; // state full
// Store match
uint32_t n = s->n_matches++;
s->l_values[n] = L;
s->m_values[n] = M;
s->d_values[n] = D;
// Store literals
uint8_t *dst = s->literals + s->n_literals;
const uint8_t *src = s->src + s->src_literal;
uint8_t *dst_end = dst + L;
if (s->src_literal + L + 16 > s->src_end) {
// Careful at the end of SRC, we can't read 16 bytes
if (L > 0)
memcpy(dst, src, L);
} else {
copy16(dst, src);
dst += 16;
src += 16;
while (dst < dst_end) {
copy16(dst, src);
dst += 16;
src += 16;
}
}
s->n_literals += L;
// Update state
s->src_literal += L + M;
return LZFSE_STATUS_OK;
}
/*! @abstract Split MATCH into one or more L,M,D parts, and push to STATE.
* @return LZFSE_STATUS_OK if OK.
* @return LZFSE_STATUS_DST_FULL if the match can't be pushed, meaning one of the
* buffers is full. In that case the state is not modified. */
static int lzfse_push_match(lzfse_encoder_state *s, const lzfse_match *match) {
// Save the initial n_matches, n_literals, src_literal
uint32_t n_matches0 = s->n_matches;
uint32_t n_literals0 = s->n_literals;
lzfse_offset src_literals0 = s->src_literal;
// L,M,D
uint32_t L = (uint32_t)(match->pos - s->src_literal); // literal count
uint32_t M = match->length; // match length
uint32_t D = (uint32_t)(match->pos - match->ref); // match distance
int ok = 1;
// Split L if too large
while (L > LZFSE_ENCODE_MAX_L_VALUE) {
if (lzfse_push_lmd(s, LZFSE_ENCODE_MAX_L_VALUE, 0, 1) != 0) {
ok = 0;
goto END;
} // take D=1 because most frequent, but not actually used
L -= LZFSE_ENCODE_MAX_L_VALUE;
}
// Split if M too large
while (M > LZFSE_ENCODE_MAX_M_VALUE) {
if (lzfse_push_lmd(s, L, LZFSE_ENCODE_MAX_M_VALUE, D) != 0) {
ok = 0;
goto END;
}
L = 0;
M -= LZFSE_ENCODE_MAX_M_VALUE;
}
// L,M in range
if (L > 0 || M > 0) {
if (lzfse_push_lmd(s, L, M, D) != 0) {
ok = 0;
goto END;
}
L = M = 0;
(void)L;
(void)M; // dead stores
}
END:
if (!ok) {
// Revert state
s->n_matches = n_matches0;
s->n_literals = n_literals0;
s->src_literal = src_literals0;
return LZFSE_STATUS_DST_FULL; // state tables full
}
return LZFSE_STATUS_OK; // OK
}
/*! @abstract Backend: add MATCH to state S. Encode block if necessary, when
* state is full.
* @return LZFSE_STATUS_OK if OK.
* @return LZFSE_STATUS_DST_FULL if the match can't be added, meaning one of the
* buffers is full. In that case the state is not modified. */
static int lzfse_backend_match(lzfse_encoder_state *s,
const lzfse_match *match) {
// Try to push the match in state
if (lzfse_push_match(s, match) == LZFSE_STATUS_OK)
return LZFSE_STATUS_OK; // OK, match added to state
// Here state tables are full, try to emit block
if (lzfse_encode_matches(s) != LZFSE_STATUS_OK)
return LZFSE_STATUS_DST_FULL; // DST full, match not added
// Here block has been emitted, re-try to push the match in state
return lzfse_push_match(s, match);
}
/*! @abstract Backend: add L literals to state S. Encode block if necessary,
* when state is full.
* @return LZFSE_STATUS_OK if OK.
* @return LZFSE_STATUS_DST_FULL if the literals can't be added, meaning one of
* the buffers is full. In that case the state is not modified. */
static int lzfse_backend_literals(lzfse_encoder_state *s, lzfse_offset L) {
// Create a fake match with M=0, D=1
lzfse_match match;
lzfse_offset pos = s->src_literal + L;
match.pos = pos;
match.ref = match.pos - 1;
match.length = 0;
return lzfse_backend_match(s, &match);
}
/*! @abstract Backend: flush final block, and emit end of stream
* @return LZFSE_STATUS_OK if OK.
* @return LZFSE_STATUS_DST_FULL if either the final block, or the end-of-stream
* can't be added, meaning one of the buffers is full. If the block was emitted,
* the state is updated to reflect this. Otherwise, it is left unchanged. */
static int lzfse_backend_end_of_stream(lzfse_encoder_state *s) {
// Final match triggers write, otherwise emit blocks when we have enough
// matches stored
if (lzfse_encode_matches(s) != LZFSE_STATUS_OK)
return LZFSE_STATUS_DST_FULL; // DST full
// Emit end-of-stream block
if (s->dst + 4 > s->dst_end)
return LZFSE_STATUS_DST_FULL; // DST full
store4(s->dst, LZFSE_ENDOFSTREAM_BLOCK_MAGIC);
s->dst += 4;
return LZFSE_STATUS_OK; // OK
}
// ===============================================================
// Encoder state management
/*! @abstract Initialize state:
* @code
* - hash table with all invalid pos, and value 0.
* - pending match to NO_MATCH.
* - src_literal to 0.
* - d_prev to 0.
@endcode
* @return LZFSE_STATUS_OK */
int lzfse_encode_init(lzfse_encoder_state *s) {
const lzfse_match NO_MATCH = {0};
lzfse_history_set line;
for (int i = 0; i < LZFSE_ENCODE_HASH_WIDTH; i++) {
line.pos[i] = -4 * LZFSE_ENCODE_MAX_D_VALUE; // invalid pos
line.value[i] = 0;
}
// Fill table
for (int i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++)
s->history_table[i] = line;
s->pending = NO_MATCH;
s->src_literal = 0;
return LZFSE_STATUS_OK; // OK
}
/*! @abstract Translate state \p src forward by \p delta > 0.
* Offsets in \p src are updated backwards to point to the same positions.
* @return LZFSE_STATUS_OK */
int lzfse_encode_translate(lzfse_encoder_state *s, lzfse_offset delta) {
assert(delta >= 0);
if (delta == 0)
return LZFSE_STATUS_OK; // OK
// SRC
s->src += delta;
// Offsets in SRC
s->src_end -= delta;
s->src_encode_i -= delta;
s->src_encode_end -= delta;
s->src_literal -= delta;
// Pending match
s->pending.pos -= delta;
s->pending.ref -= delta;
// history_table positions, translated, and clamped to invalid pos
int32_t invalidPos = -4 * LZFSE_ENCODE_MAX_D_VALUE;
for (int i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++) {
int32_t *p = &(s->history_table[i].pos[0]);
for (int j = 0; j < LZFSE_ENCODE_HASH_WIDTH; j++) {
lzfse_offset newPos = p[j] - delta; // translate
p[j] = (int32_t)((newPos < invalidPos) ? invalidPos : newPos); // clamp
}
}
return LZFSE_STATUS_OK; // OK
}
// ===============================================================
// Encoder front end
int lzfse_encode_base(lzfse_encoder_state *s) {
lzfse_history_set *history_table = s->history_table;
lzfse_history_set *hashLine = 0;
lzfse_history_set newH;
const lzfse_match NO_MATCH = {0};
int ok = 1;
memset(&newH, 0x00, sizeof(newH));
// 8 byte padding at end of buffer
s->src_encode_end = s->src_end - 8;
for (; s->src_encode_i < s->src_encode_end; s->src_encode_i++) {
lzfse_offset pos = s->src_encode_i; // pos >= 0
// Load 4 byte value and get hash line
uint32_t x = load4(s->src + pos);
hashLine = history_table + hashX(x);
lzfse_history_set h = *hashLine;
// Prepare next hash line (component 0 is the most recent) to prepare new
// entries (stored later)
{
newH.pos[0] = (int32_t)pos;
for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++)
newH.pos[k + 1] = h.pos[k];
newH.value[0] = x;
for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++)
newH.value[k + 1] = h.value[k];
}
// Do not look for a match if we are still covered by a previous match
if (pos < s->src_literal)
goto END_POS;
// Search best incoming match
lzfse_match incoming = {.pos = pos, .ref = 0, .length = 0};
// Check for matches. We consider matches of length >= 4 only.
for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH; k++) {
uint32_t d = h.value[k] ^ x;
if (d)
continue; // no 4 byte match
int32_t ref = h.pos[k];
if (ref + LZFSE_ENCODE_MAX_D_VALUE < pos)
continue; // too far
const uint8_t *src_ref = s->src + ref;
const uint8_t *src_pos = s->src + pos;
uint32_t length = 4;
uint32_t maxLength =
(uint32_t)(s->src_end - pos - 8); // ensure we don't hit the end of SRC
while (length < maxLength) {
uint64_t d = load8(src_ref + length) ^ load8(src_pos + length);
if (d == 0) {
length += 8;
continue;
}
length +=
(__builtin_ctzll(d) >> 3); // ctzll must be called only with D != 0
break;
}
if (length > incoming.length) {
incoming.length = length;
incoming.ref = ref;
} // keep if longer
}
// No incoming match?
if (incoming.length == 0) {
// We may still want to emit some literals here, to not lag too far behind
// the current search point, and avoid
// ending up with a literal block not fitting in the state.
lzfse_offset n_literals = pos - s->src_literal;
// The threshold here should be larger than a couple of MAX_L_VALUE, and
// much smaller than LITERALS_PER_BLOCK
if (n_literals > 8 * LZFSE_ENCODE_MAX_L_VALUE) {
// Here, we need to consume some literals. Emit pending match if there
// is one
if (s->pending.length > 0) {
if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) {
ok = 0;
goto END;
}
s->pending = NO_MATCH;
} else {
// No pending match, emit a full LZFSE_ENCODE_MAX_L_VALUE block of
// literals
if (lzfse_backend_literals(s, LZFSE_ENCODE_MAX_L_VALUE) !=
LZFSE_STATUS_OK) {
ok = 0;
goto END;
}
}
}
goto END_POS; // no incoming match
}
// Limit match length (it may still be expanded backwards, but this is
// bounded by the limit on literals we tested before)
if (incoming.length > LZFSE_ENCODE_MAX_MATCH_LENGTH) {
incoming.length = LZFSE_ENCODE_MAX_MATCH_LENGTH;
}
// Expand backwards (since this is expensive, we do this for the best match
// only)
while (incoming.pos > s->src_literal && incoming.ref > 0 &&
s->src[incoming.ref - 1] == s->src[incoming.pos - 1]) {
incoming.pos--;
incoming.ref--;
}
incoming.length += pos - incoming.pos; // update length after expansion
// Match filtering heuristic (from LZVN). INCOMING is always defined here.
// Incoming is 'good', emit incoming
if (incoming.length >= LZFSE_ENCODE_GOOD_MATCH) {
if (lzfse_backend_match(s, &incoming) != LZFSE_STATUS_OK) {
ok = 0;
goto END;
}
s->pending = NO_MATCH;
goto END_POS;
}
// No pending, keep incoming
if (s->pending.length == 0) {
s->pending = incoming;
goto END_POS;
}
// No overlap, emit pending, keep incoming
if (s->pending.pos + s->pending.length <= incoming.pos) {
if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) {
ok = 0;
goto END;
}
s->pending = incoming;
goto END_POS;
}
// Overlap: emit longest
if (incoming.length > s->pending.length) {
if (lzfse_backend_match(s, &incoming) != LZFSE_STATUS_OK) {
ok = 0;
goto END;
}
} else {
if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) {
ok = 0;
goto END;
}
}
s->pending = NO_MATCH;
END_POS:
// We are done with this src_encode_i.
// Update state now (s->pending has already been updated).
*hashLine = newH;
}
END:
return ok ? LZFSE_STATUS_OK : LZFSE_STATUS_DST_FULL;
}
int lzfse_encode_finish(lzfse_encoder_state *s) {
const lzfse_match NO_MATCH = {0};
// Emit pending match
if (s->pending.length > 0) {
if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK)
return LZFSE_STATUS_DST_FULL;
s->pending = NO_MATCH;
}
// Emit final literals if any
lzfse_offset L = s->src_end - s->src_literal;
if (L > 0) {
if (lzfse_backend_literals(s, L) != LZFSE_STATUS_OK)
return LZFSE_STATUS_DST_FULL;
}
// Emit all matches, and end-of-stream block
if (lzfse_backend_end_of_stream(s) != LZFSE_STATUS_OK)
return LZFSE_STATUS_DST_FULL;
return LZFSE_STATUS_OK;
}
+218
查看文件
@@ -0,0 +1,218 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LZFSE_ENCODE_TABLES_H
#define LZFSE_ENCODE_TABLES_H
#if defined(_MSC_VER) && !defined(__clang__)
# define inline __inline
#endif
static inline uint8_t l_base_from_value(int32_t value) {
static const uint8_t sym[LZFSE_ENCODE_MAX_L_VALUE + 1] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16,
16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19};
return sym[value];
}
static inline uint8_t m_base_from_value(int32_t value) {
static const uint8_t sym[LZFSE_ENCODE_MAX_M_VALUE + 1] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16,
16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19};
return sym[value];
}
static inline uint8_t d_base_from_value(int32_t value) {
static const uint8_t sym[64 * 4] = {
0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9,
9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12,
13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15,
15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, 17, 18, 19, 20, 20, 21, 21,
22, 22, 23, 23, 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27,
27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 29, 29,
30, 30, 30, 30, 30, 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32,
32, 32, 32, 33, 34, 35, 36, 36, 37, 37, 38, 38, 39, 39, 40, 40, 40, 40,
41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 44, 44,
44, 44, 45, 45, 45, 45, 45, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46,
47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 49, 50, 51, 52, 52,
53, 53, 54, 54, 55, 55, 56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58,
59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, 61, 61, 61, 61, 61, 61,
61, 61, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63,
0, 0, 0, 0};
int index = 0;
int in_range_k;
in_range_k = (value >= 0 && value < 60);
index |= (((value - 0) >> 0) + 0) & -in_range_k;
in_range_k = (value >= 60 && value < 1020);
index |= (((value - 60) >> 4) + 64) & -in_range_k;
in_range_k = (value >= 1020 && value < 16380);
index |= (((value - 1020) >> 8) + 128) & -in_range_k;
in_range_k = (value >= 16380 && value < 262140);
index |= (((value - 16380) >> 12) + 192) & -in_range_k;
return sym[index & 255];
}
#endif // LZFSE_ENCODE_TABLES_H
+216
查看文件
@@ -0,0 +1,216 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "lzfse_internal.h"
// Initialize encoder table T[NSYMBOLS].
// NSTATES = sum FREQ[i] is the number of states (a power of 2)
// NSYMBOLS is the number of symbols.
// FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i]
// >= 0.
// Some symbols may have a 0 frequency. In that case, they should not be
// present in the data.
void fse_init_encoder_table(int nstates, int nsymbols,
const uint16_t *__restrict freq,
fse_encoder_entry *__restrict t) {
int offset = 0; // current offset
int n_clz = __builtin_clz(nstates);
for (int i = 0; i < nsymbols; i++) {
int f = (int)freq[i];
if (f == 0)
continue; // skip this symbol, no occurrences
int k =
__builtin_clz(f) - n_clz; // shift needed to ensure N <= (F<<K) < 2*N
t[i].s0 = (int16_t)((f << k) - nstates);
t[i].k = (int16_t)k;
t[i].delta0 = (int16_t)(offset - f + (nstates >> k));
t[i].delta1 = (int16_t)(offset - f + (nstates >> (k - 1)));
offset += f;
}
}
// Initialize decoder table T[NSTATES].
// NSTATES = sum FREQ[i] is the number of states (a power of 2)
// NSYMBOLS is the number of symbols.
// FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i]
// >= 0.
// Some symbols may have a 0 frequency. In that case, they should not be
// present in the data.
int fse_init_decoder_table(int nstates, int nsymbols,
const uint16_t *__restrict freq,
int32_t *__restrict t) {
assert(nsymbols <= 256);
assert(fse_check_freq(freq, nsymbols, nstates) == 0);
int n_clz = __builtin_clz(nstates);
int sum_of_freq = 0;
for (int i = 0; i < nsymbols; i++) {
int f = (int)freq[i];
if (f == 0)
continue; // skip this symbol, no occurrences
sum_of_freq += f;
if (sum_of_freq > nstates) {
return -1;
}
int k =
__builtin_clz(f) - n_clz; // shift needed to ensure N <= (F<<K) < 2*N
int j0 = ((2 * nstates) >> k) - f;
// Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F
for (int j = 0; j < f; j++) {
fse_decoder_entry e;
e.symbol = (uint8_t)i;
if (j < j0) {
e.k = (int8_t)k;
e.delta = (int16_t)(((f + j) << k) - nstates);
} else {
e.k = (int8_t)(k - 1);
e.delta = (int16_t)((j - j0) << (k - 1));
}
memcpy(t, &e, sizeof(e));
t++;
}
}
return 0; // OK
}
// Initialize value decoder table T[NSTATES].
// NSTATES = sum FREQ[i] is the number of states (a power of 2)
// NSYMBOLS is the number of symbols.
// FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i]
// >= 0.
// SYMBOL_VBITS[NSYMBOLS] and SYMBOLS_VBASE[NSYMBOLS] are the number of value
// bits to read and the base value for each symbol.
// Some symbols may have a 0 frequency. In that case, they should not be
// present in the data.
void fse_init_value_decoder_table(int nstates, int nsymbols,
const uint16_t *__restrict freq,
const uint8_t *__restrict symbol_vbits,
const int32_t *__restrict symbol_vbase,
fse_value_decoder_entry *__restrict t) {
assert(nsymbols <= 256);
assert(fse_check_freq(freq, nsymbols, nstates) == 0);
int n_clz = __builtin_clz(nstates);
for (int i = 0; i < nsymbols; i++) {
int f = (int)freq[i];
if (f == 0)
continue; // skip this symbol, no occurrences
int k =
__builtin_clz(f) - n_clz; // shift needed to ensure N <= (F<<K) < 2*N
int j0 = ((2 * nstates) >> k) - f;
fse_value_decoder_entry ei = {0};
ei.value_bits = symbol_vbits[i];
ei.vbase = symbol_vbase[i];
// Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F
for (int j = 0; j < f; j++) {
fse_value_decoder_entry e = ei;
if (j < j0) {
e.total_bits = (uint8_t)k + e.value_bits;
e.delta = (int16_t)(((f + j) << k) - nstates);
} else {
e.total_bits = (uint8_t)(k - 1) + e.value_bits;
e.delta = (int16_t)((j - j0) << (k - 1));
}
memcpy(t, &e, 8);
t++;
}
}
}
// Remove states from symbols until the correct number of states is used.
static void fse_adjust_freqs(uint16_t *freq, int overrun, int nsymbols) {
for (int shift = 3; overrun != 0; shift--) {
for (int sym = 0; sym < nsymbols; sym++) {
if (freq[sym] > 1) {
int n = (freq[sym] - 1) >> shift;
if (n > overrun)
n = overrun;
freq[sym] -= n;
overrun -= n;
if (overrun == 0)
break;
}
}
}
}
// Normalize a table T[NSYMBOLS] of occurrences to FREQ[NSYMBOLS].
void fse_normalize_freq(int nstates, int nsymbols, const uint32_t *__restrict t,
uint16_t *__restrict freq) {
uint32_t s_count = 0;
int remaining = nstates; // must be signed; this may become < 0
int max_freq = 0;
int max_freq_sym = 0;
int shift = __builtin_clz(nstates) - 1;
uint32_t highprec_step;
// Compute the total number of symbol occurrences
for (int i = 0; i < nsymbols; i++)
s_count += t[i];
if (s_count == 0)
highprec_step = 0; // no symbols used
else
highprec_step = ((uint32_t)1 << 31) / s_count;
for (int i = 0; i < nsymbols; i++) {
// Rescale the occurrence count to get the normalized frequency.
// Round up if the fractional part is >= 0.5; otherwise round down.
// For efficiency, we do this calculation using integer arithmetic.
int f = (((t[i] * highprec_step) >> shift) + 1) >> 1;
// If a symbol was used, it must be given a nonzero normalized frequency.
if (f == 0 && t[i] != 0)
f = 1;
freq[i] = f;
remaining -= f;
// Remember the maximum frequency and which symbol had it.
if (f > max_freq) {
max_freq = f;
max_freq_sym = i;
}
}
// If there remain states to be assigned, then just assign them to the most
// frequent symbol. Alternatively, if we assigned more states than were
// actually available, then either remove states from the most frequent symbol
// (for minor overruns) or use the slower adjustment algorithm (for major
// overruns).
if (-remaining < (max_freq >> 2)) {
freq[max_freq_sym] += remaining;
} else {
fse_adjust_freqs(freq, -remaining, nsymbols);
}
}
+631
查看文件
@@ -0,0 +1,631 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// Finite state entropy coding (FSE)
// This is an implementation of the tANS algorithm described by Jarek Duda,
// we use the more descriptive name "Finite State Entropy".
#pragma once
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
// Select between 32/64-bit I/O streams for FSE. Note that the FSE stream
// size need not match the word size of the machine, but in practice you
// want to use 64b streams on 64b systems for better performance.
#if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__)
#define FSE_IOSTREAM_64 1
#else
#define FSE_IOSTREAM_64 0
#endif
#if defined(_MSC_VER) && !defined(__clang__)
# define FSE_INLINE __forceinline
# define inline __inline
# pragma warning(disable : 4068) // warning C4068: unknown pragma
#else
# define FSE_INLINE static inline __attribute__((__always_inline__))
#endif
// MARK: - Bit utils
/*! @abstract Signed type used to represent bit count. */
typedef int32_t fse_bit_count;
/*! @abstract Unsigned type used to represent FSE state. */
typedef uint16_t fse_state;
// Mask the NBITS lsb of X. 0 <= NBITS < 64
static inline uint64_t fse_mask_lsb64(uint64_t x, fse_bit_count nbits) {
static const uint64_t mtable[65] = {
0x0000000000000000LLU, 0x0000000000000001LLU, 0x0000000000000003LLU,
0x0000000000000007LLU, 0x000000000000000fLLU, 0x000000000000001fLLU,
0x000000000000003fLLU, 0x000000000000007fLLU, 0x00000000000000ffLLU,
0x00000000000001ffLLU, 0x00000000000003ffLLU, 0x00000000000007ffLLU,
0x0000000000000fffLLU, 0x0000000000001fffLLU, 0x0000000000003fffLLU,
0x0000000000007fffLLU, 0x000000000000ffffLLU, 0x000000000001ffffLLU,
0x000000000003ffffLLU, 0x000000000007ffffLLU, 0x00000000000fffffLLU,
0x00000000001fffffLLU, 0x00000000003fffffLLU, 0x00000000007fffffLLU,
0x0000000000ffffffLLU, 0x0000000001ffffffLLU, 0x0000000003ffffffLLU,
0x0000000007ffffffLLU, 0x000000000fffffffLLU, 0x000000001fffffffLLU,
0x000000003fffffffLLU, 0x000000007fffffffLLU, 0x00000000ffffffffLLU,
0x00000001ffffffffLLU, 0x00000003ffffffffLLU, 0x00000007ffffffffLLU,
0x0000000fffffffffLLU, 0x0000001fffffffffLLU, 0x0000003fffffffffLLU,
0x0000007fffffffffLLU, 0x000000ffffffffffLLU, 0x000001ffffffffffLLU,
0x000003ffffffffffLLU, 0x000007ffffffffffLLU, 0x00000fffffffffffLLU,
0x00001fffffffffffLLU, 0x00003fffffffffffLLU, 0x00007fffffffffffLLU,
0x0000ffffffffffffLLU, 0x0001ffffffffffffLLU, 0x0003ffffffffffffLLU,
0x0007ffffffffffffLLU, 0x000fffffffffffffLLU, 0x001fffffffffffffLLU,
0x003fffffffffffffLLU, 0x007fffffffffffffLLU, 0x00ffffffffffffffLLU,
0x01ffffffffffffffLLU, 0x03ffffffffffffffLLU, 0x07ffffffffffffffLLU,
0x0fffffffffffffffLLU, 0x1fffffffffffffffLLU, 0x3fffffffffffffffLLU,
0x7fffffffffffffffLLU, 0xffffffffffffffffLLU,
};
return x & mtable[nbits];
}
// Mask the NBITS lsb of X. 0 <= NBITS < 32
static inline uint32_t fse_mask_lsb32(uint32_t x, fse_bit_count nbits) {
static const uint32_t mtable[33] = {
0x0000000000000000U, 0x0000000000000001U, 0x0000000000000003U,
0x0000000000000007U, 0x000000000000000fU, 0x000000000000001fU,
0x000000000000003fU, 0x000000000000007fU, 0x00000000000000ffU,
0x00000000000001ffU, 0x00000000000003ffU, 0x00000000000007ffU,
0x0000000000000fffU, 0x0000000000001fffU, 0x0000000000003fffU,
0x0000000000007fffU, 0x000000000000ffffU, 0x000000000001ffffU,
0x000000000003ffffU, 0x000000000007ffffU, 0x00000000000fffffU,
0x00000000001fffffU, 0x00000000003fffffU, 0x00000000007fffffU,
0x0000000000ffffffU, 0x0000000001ffffffU, 0x0000000003ffffffU,
0x0000000007ffffffU, 0x000000000fffffffU, 0x000000001fffffffU,
0x000000003fffffffU, 0x000000007fffffffU, 0x00000000ffffffffU,
};
return x & mtable[nbits];
}
/*! @abstract Select \c nbits at index \c start from \c x.
* 0 <= start <= start+nbits <= 64 */
FSE_INLINE uint64_t fse_extract_bits64(uint64_t x, fse_bit_count start,
fse_bit_count nbits) {
#if defined(__GNUC__)
// If START and NBITS are constants, map to bit-field extraction instructions
if (__builtin_constant_p(start) && __builtin_constant_p(nbits))
return (x >> start) & ((1LLU << nbits) - 1LLU);
#endif
// Otherwise, shift and mask
return fse_mask_lsb64(x >> start, nbits);
}
/*! @abstract Select \c nbits at index \c start from \c x.
* 0 <= start <= start+nbits <= 32 */
FSE_INLINE uint32_t fse_extract_bits32(uint32_t x, fse_bit_count start,
fse_bit_count nbits) {
#if defined(__GNUC__)
// If START and NBITS are constants, map to bit-field extraction instructions
if (__builtin_constant_p(start) && __builtin_constant_p(nbits))
return (x >> start) & ((1U << nbits) - 1U);
#endif
// Otherwise, shift and mask
return fse_mask_lsb32(x >> start, nbits);
}
// MARK: - Bit stream
// I/O streams
// The streams can be shared between several FSE encoders/decoders, which is why
// they are not in the state struct
/*! @abstract Output stream, 64-bit accum. */
typedef struct {
uint64_t accum; // Output bits
fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0
} fse_out_stream64;
/*! @abstract Output stream, 32-bit accum. */
typedef struct {
uint32_t accum; // Output bits
fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0
} fse_out_stream32;
/*! @abstract Object representing an input stream. */
typedef struct {
uint64_t accum; // Input bits
fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0
} fse_in_stream64;
/*! @abstract Object representing an input stream. */
typedef struct {
uint32_t accum; // Input bits
fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0
} fse_in_stream32;
/*! @abstract Initialize an output stream object. */
FSE_INLINE void fse_out_init64(fse_out_stream64 *s) {
s->accum = 0;
s->accum_nbits = 0;
}
/*! @abstract Initialize an output stream object. */
FSE_INLINE void fse_out_init32(fse_out_stream32 *s) {
s->accum = 0;
s->accum_nbits = 0;
}
/*! @abstract Write full bytes from the accumulator to output buffer, ensuring
* accum_nbits is in [0, 7].
* We assume we can write 8 bytes to the output buffer \c (*pbuf[0..7]) in all
* cases.
* @note *pbuf is incremented by the number of written bytes. */
FSE_INLINE void fse_out_flush64(fse_out_stream64 *s, uint8_t **pbuf) {
fse_bit_count nbits =
s->accum_nbits & -8; // number of bits written, multiple of 8
// Write 8 bytes of current accumulator
memcpy(*pbuf, &(s->accum), 8);
*pbuf += (nbits >> 3); // bytes
// Update state
s->accum >>= nbits; // remove nbits
s->accum_nbits -= nbits;
assert(s->accum_nbits >= 0 && s->accum_nbits <= 7);
assert(s->accum_nbits == 64 || (s->accum >> s->accum_nbits) == 0);
}
/*! @abstract Write full bytes from the accumulator to output buffer, ensuring
* accum_nbits is in [0, 7].
* We assume we can write 4 bytes to the output buffer \c (*pbuf[0..3]) in all
* cases.
* @note *pbuf is incremented by the number of written bytes. */
FSE_INLINE void fse_out_flush32(fse_out_stream32 *s, uint8_t **pbuf) {
fse_bit_count nbits =
s->accum_nbits & -8; // number of bits written, multiple of 8
// Write 4 bytes of current accumulator
memcpy(*pbuf, &(s->accum), 4);
*pbuf += (nbits >> 3); // bytes
// Update state
s->accum >>= nbits; // remove nbits
s->accum_nbits -= nbits;
assert(s->accum_nbits >= 0 && s->accum_nbits <= 7);
assert(s->accum_nbits == 32 || (s->accum >> s->accum_nbits) == 0);
}
/*! @abstract Write the last bytes from the accumulator to output buffer,
* ensuring accum_nbits is in [-7, 0]. Bits are padded with 0 if needed.
* We assume we can write 8 bytes to the output buffer \c (*pbuf[0..7]) in all
* cases.
* @note *pbuf is incremented by the number of written bytes. */
FSE_INLINE void fse_out_finish64(fse_out_stream64 *s, uint8_t **pbuf) {
fse_bit_count nbits =
(s->accum_nbits + 7) & -8; // number of bits written, multiple of 8
// Write 8 bytes of current accumulator
memcpy(*pbuf, &(s->accum), 8);
*pbuf += (nbits >> 3); // bytes
// Update state
s->accum = 0; // remove nbits
s->accum_nbits -= nbits;
assert(s->accum_nbits >= -7 && s->accum_nbits <= 0);
}
/*! @abstract Write the last bytes from the accumulator to output buffer,
* ensuring accum_nbits is in [-7, 0]. Bits are padded with 0 if needed.
* We assume we can write 4 bytes to the output buffer \c (*pbuf[0..3]) in all
* cases.
* @note *pbuf is incremented by the number of written bytes. */
FSE_INLINE void fse_out_finish32(fse_out_stream32 *s, uint8_t **pbuf) {
fse_bit_count nbits =
(s->accum_nbits + 7) & -8; // number of bits written, multiple of 8
// Write 8 bytes of current accumulator
memcpy(*pbuf, &(s->accum), 4);
*pbuf += (nbits >> 3); // bytes
// Update state
s->accum = 0; // remove nbits
s->accum_nbits -= nbits;
assert(s->accum_nbits >= -7 && s->accum_nbits <= 0);
}
/*! @abstract Accumulate \c n bits \c b to output stream \c s. We \b must have:
* 0 <= b < 2^n, and N + s->accum_nbits <= 64.
* @note The caller must ensure out_flush is called \b before the accumulator
* overflows to more than 64 bits. */
FSE_INLINE void fse_out_push64(fse_out_stream64 *s, fse_bit_count n,
uint64_t b) {
s->accum |= b << s->accum_nbits;
s->accum_nbits += n;
assert(s->accum_nbits >= 0 && s->accum_nbits <= 64);
assert(s->accum_nbits == 64 || (s->accum >> s->accum_nbits) == 0);
}
/*! @abstract Accumulate \c n bits \c b to output stream \c s. We \b must have:
* 0 <= n < 2^n, and n + s->accum_nbits <= 32.
* @note The caller must ensure out_flush is called \b before the accumulator
* overflows to more than 32 bits. */
FSE_INLINE void fse_out_push32(fse_out_stream32 *s, fse_bit_count n,
uint32_t b) {
s->accum |= b << s->accum_nbits;
s->accum_nbits += n;
assert(s->accum_nbits >= 0 && s->accum_nbits <= 32);
assert(s->accum_nbits == 32 || (s->accum >> s->accum_nbits) == 0);
}
#if FSE_IOSTREAM_64
#define DEBUG_CHECK_INPUT_STREAM_PARAMETERS \
assert(s->accum_nbits >= 56 && s->accum_nbits < 64); \
assert((s->accum >> s->accum_nbits) == 0);
#else
#define DEBUG_CHECK_INPUT_STREAM_PARAMETERS \
assert(s->accum_nbits >= 24 && s->accum_nbits < 32); \
assert((s->accum >> s->accum_nbits) == 0);
#endif
/*! @abstract Initialize the fse input stream so that accum holds between 56
* and 63 bits. We never want to have 64 bits in the stream, because that allows
* us to avoid a special case in the fse_in_pull function (eliminating an
* unpredictable branch), while not requiring any additional fse_flush
* operations. This is why we have the special case for n == 0 (in which case
* we want to load only 7 bytes instead of 8). */
FSE_INLINE int fse_in_checked_init64(fse_in_stream64 *s, fse_bit_count n,
const uint8_t **pbuf,
const uint8_t *buf_start) {
if (n) {
if (*pbuf < buf_start + 8)
return -1; // out of range
*pbuf -= 8;
memcpy(&(s->accum), *pbuf, 8);
s->accum_nbits = n + 64;
} else {
if (*pbuf < buf_start + 7)
return -1; // out of range
*pbuf -= 7;
memcpy(&(s->accum), *pbuf, 7);
s->accum &= 0xffffffffffffff;
s->accum_nbits = n + 56;
}
if ((s->accum_nbits < 56 || s->accum_nbits >= 64) ||
((s->accum >> s->accum_nbits) != 0)) {
return -1; // the incoming input is wrong (encoder should have zeroed the
// upper bits)
}
return 0; // OK
}
/*! @abstract Identical to previous function, but for 32-bit operation
* (resulting bit count is between 24 and 31 bits). */
FSE_INLINE int fse_in_checked_init32(fse_in_stream32 *s, fse_bit_count n,
const uint8_t **pbuf,
const uint8_t *buf_start) {
if (n) {
if (*pbuf < buf_start + 4)
return -1; // out of range
*pbuf -= 4;
memcpy(&(s->accum), *pbuf, 4);
s->accum_nbits = n + 32;
} else {
if (*pbuf < buf_start + 3)
return -1; // out of range
*pbuf -= 3;
memcpy(&(s->accum), *pbuf, 3);
s->accum &= 0xffffff;
s->accum_nbits = n + 24;
}
if ((s->accum_nbits < 24 || s->accum_nbits >= 32) ||
((s->accum >> s->accum_nbits) != 0)) {
return -1; // the incoming input is wrong (encoder should have zeroed the
// upper bits)
}
return 0; // OK
}
/*! @abstract Read in new bytes from buffer to ensure that we have a full
* complement of bits in the stream object (again, between 56 and 63 bits).
* checking the new value of \c *pbuf remains >= \c buf_start.
* @return 0 if OK.
* @return -1 on failure. */
FSE_INLINE int fse_in_checked_flush64(fse_in_stream64 *s, const uint8_t **pbuf,
const uint8_t *buf_start) {
// Get number of bits to add to bring us into the desired range.
fse_bit_count nbits = (63 - s->accum_nbits) & -8;
// Convert bits to bytes and decrement buffer address, then load new data.
const uint8_t *buf = (*pbuf) - (nbits >> 3);
if (buf < buf_start) {
return -1; // out of range
}
*pbuf = buf;
uint64_t incoming;
memcpy(&incoming, buf, 8);
// Update the state object and verify its validity (in DEBUG).
s->accum = (s->accum << nbits) | fse_mask_lsb64(incoming, nbits);
s->accum_nbits += nbits;
DEBUG_CHECK_INPUT_STREAM_PARAMETERS
return 0; // OK
}
/*! @abstract Identical to previous function (but again, we're only filling
* a 32-bit field with between 24 and 31 bits). */
FSE_INLINE int fse_in_checked_flush32(fse_in_stream32 *s, const uint8_t **pbuf,
const uint8_t *buf_start) {
// Get number of bits to add to bring us into the desired range.
fse_bit_count nbits = (31 - s->accum_nbits) & -8;
if (nbits > 0) {
// Convert bits to bytes and decrement buffer address, then load new data.
const uint8_t *buf = (*pbuf) - (nbits >> 3);
if (buf < buf_start) {
return -1; // out of range
}
*pbuf = buf;
uint32_t incoming = *((uint32_t *)buf);
// Update the state object and verify its validity (in DEBUG).
s->accum = (s->accum << nbits) | fse_mask_lsb32(incoming, nbits);
s->accum_nbits += nbits;
}
DEBUG_CHECK_INPUT_STREAM_PARAMETERS
return 0; // OK
}
/*! @abstract Pull n bits out of the fse stream object. */
FSE_INLINE uint64_t fse_in_pull64(fse_in_stream64 *s, fse_bit_count n) {
assert(n >= 0 && n <= s->accum_nbits);
s->accum_nbits -= n;
uint64_t result = s->accum >> s->accum_nbits;
s->accum = fse_mask_lsb64(s->accum, s->accum_nbits);
return result;
}
/*! @abstract Pull n bits out of the fse stream object. */
FSE_INLINE uint32_t fse_in_pull32(fse_in_stream32 *s, fse_bit_count n) {
assert(n >= 0 && n <= s->accum_nbits);
s->accum_nbits -= n;
uint32_t result = s->accum >> s->accum_nbits;
s->accum = fse_mask_lsb32(s->accum, s->accum_nbits);
return result;
}
// MARK: - Encode/Decode
// Map to 32/64-bit implementations and types for I/O
#if FSE_IOSTREAM_64
typedef uint64_t fse_bits;
typedef fse_out_stream64 fse_out_stream;
typedef fse_in_stream64 fse_in_stream;
#define fse_mask_lsb fse_mask_lsb64
#define fse_extract_bits fse_extract_bits64
#define fse_out_init fse_out_init64
#define fse_out_flush fse_out_flush64
#define fse_out_finish fse_out_finish64
#define fse_out_push fse_out_push64
#define fse_in_init fse_in_checked_init64
#define fse_in_checked_init fse_in_checked_init64
#define fse_in_flush fse_in_checked_flush64
#define fse_in_checked_flush fse_in_checked_flush64
#define fse_in_flush2(_unused, _parameters, _unused2) 0 /* nothing */
#define fse_in_checked_flush2(_unused, _parameters) /* nothing */
#define fse_in_pull fse_in_pull64
#else
typedef uint32_t fse_bits;
typedef fse_out_stream32 fse_out_stream;
typedef fse_in_stream32 fse_in_stream;
#define fse_mask_lsb fse_mask_lsb32
#define fse_extract_bits fse_extract_bits32
#define fse_out_init fse_out_init32
#define fse_out_flush fse_out_flush32
#define fse_out_finish fse_out_finish32
#define fse_out_push fse_out_push32
#define fse_in_init fse_in_checked_init32
#define fse_in_checked_init fse_in_checked_init32
#define fse_in_flush fse_in_checked_flush32
#define fse_in_checked_flush fse_in_checked_flush32
#define fse_in_flush2 fse_in_checked_flush32
#define fse_in_checked_flush2 fse_in_checked_flush32
#define fse_in_pull fse_in_pull32
#endif
/*! @abstract Entry for one symbol in the encoder table (64b). */
typedef struct {
int16_t s0; // First state requiring a K-bit shift
int16_t k; // States S >= S0 are shifted K bits. States S < S0 are
// shifted K-1 bits
int16_t delta0; // Relative increment used to compute next state if S >= S0
int16_t delta1; // Relative increment used to compute next state if S < S0
} fse_encoder_entry;
/*! @abstract Entry for one state in the decoder table (32b). */
typedef struct { // DO NOT REORDER THE FIELDS
int8_t k; // Number of bits to read
uint8_t symbol; // Emitted symbol
int16_t delta; // Signed increment used to compute next state (+bias)
} fse_decoder_entry;
/*! @abstract Entry for one state in the value decoder table (64b). */
typedef struct { // DO NOT REORDER THE FIELDS
uint8_t total_bits; // state bits + extra value bits = shift for next decode
uint8_t value_bits; // extra value bits
int16_t delta; // state base (delta)
int32_t vbase; // value base
} fse_value_decoder_entry;
/*! @abstract Encode SYMBOL using the encoder table, and update \c *pstate,
* \c out.
* @note The caller must ensure we have enough bits available in the output
* stream accumulator. */
FSE_INLINE void fse_encode(fse_state *__restrict pstate,
const fse_encoder_entry *__restrict encoder_table,
fse_out_stream *__restrict out, uint8_t symbol) {
int s = *pstate;
fse_encoder_entry e = encoder_table[symbol];
int s0 = e.s0;
int k = e.k;
int delta0 = e.delta0;
int delta1 = e.delta1;
// Number of bits to write
int hi = s >= s0;
fse_bit_count nbits = hi ? k : (k - 1);
fse_state delta = hi ? delta0 : delta1;
// Write lower NBITS of state
fse_bits b = fse_mask_lsb(s, nbits);
fse_out_push(out, nbits, b);
// Update state with remaining bits and delta
*pstate = delta + (s >> nbits);
}
/*! @abstract Decode and return symbol using the decoder table, and update
* \c *pstate, \c in.
* @note The caller must ensure we have enough bits available in the input
* stream accumulator. */
FSE_INLINE uint8_t fse_decode(fse_state *__restrict pstate,
const int32_t *__restrict decoder_table,
fse_in_stream *__restrict in) {
int32_t e = decoder_table[*pstate];
// Update state from K bits of input + DELTA
*pstate = (fse_state)(e >> 16) + (fse_state)fse_in_pull(in, e & 0xff);
// Return the symbol for this state
return fse_extract_bits(e, 8, 8); // symbol
}
/*! @abstract Decode and return value using the decoder table, and update \c
* *pstate, \c in.
* \c value_decoder_table[nstates]
* @note The caller must ensure we have enough bits available in the input
* stream accumulator. */
FSE_INLINE int32_t
fse_value_decode(fse_state *__restrict pstate,
const fse_value_decoder_entry *value_decoder_table,
fse_in_stream *__restrict in) {
fse_value_decoder_entry entry = value_decoder_table[*pstate];
uint32_t state_and_value_bits = (uint32_t)fse_in_pull(in, entry.total_bits);
*pstate =
(fse_state)(entry.delta + (state_and_value_bits >> entry.value_bits));
return (int32_t)(entry.vbase +
fse_mask_lsb(state_and_value_bits, entry.value_bits));
}
// MARK: - Tables
// IMPORTANT: To properly decode an FSE encoded stream, both encoder/decoder
// tables shall be initialized with the same parameters, including the
// FREQ[NSYMBOL] array.
//
/*! @abstract Sanity check on frequency table, verify sum of \c freq
* is <= \c number_of_states. */
FSE_INLINE int fse_check_freq(const uint16_t *freq_table,
const size_t table_size,
const size_t number_of_states) {
size_t sum_of_freq = 0;
for (int i = 0; i < table_size; i++) {
sum_of_freq += freq_table[i];
}
return (sum_of_freq > number_of_states) ? -1 : 0;
}
/*! @abstract Initialize encoder table \c t[nsymbols].
*
* @param nstates
* sum \c freq[i]; the number of states (a power of 2).
*
* @param nsymbols
* the number of symbols.
*
* @param freq[nsymbols]
* is a normalized histogram of symbol frequencies, with \c freq[i] >= 0.
* Some symbols may have a 0 frequency. In that case they should not be
* present in the data.
*/
void fse_init_encoder_table(int nstates, int nsymbols,
const uint16_t *__restrict freq,
fse_encoder_entry *__restrict t);
/*! @abstract Initialize decoder table \c t[nstates].
*
* @param nstates
* sum \c freq[i]; the number of states (a power of 2).
*
* @param nsymbols
* the number of symbols.
*
* @param feq[nsymbols]
* a normalized histogram of symbol frequencies, with \c freq[i] >= 0.
* Some symbols may have a 0 frequency. In that case they should not be
* present in the data.
*
* @return 0 if OK.
* @return -1 on failure.
*/
int fse_init_decoder_table(int nstates, int nsymbols,
const uint16_t *__restrict freq,
int32_t *__restrict t);
/*! @abstract Initialize value decoder table \c t[nstates].
*
* @param nstates
* sum \cfreq[i]; the number of states (a power of 2).
*
* @param nsymbols
* the number of symbols.
*
* @param freq[nsymbols]
* a normalized histogram of symbol frequencies, with \c freq[i] >= 0.
* \c symbol_vbits[nsymbols] and \c symbol_vbase[nsymbols] are the number of
* value bits to read and the base value for each symbol.
* Some symbols may have a 0 frequency. In that case they should not be
* present in the data.
*/
void fse_init_value_decoder_table(int nstates, int nsymbols,
const uint16_t *__restrict freq,
const uint8_t *__restrict symbol_vbits,
const int32_t *__restrict symbol_vbase,
fse_value_decoder_entry *__restrict t);
/*! @abstract Normalize a table \c t[nsymbols] of occurrences to
* \c freq[nsymbols]. */
void fse_normalize_freq(int nstates, int nsymbols, const uint32_t *__restrict t,
uint16_t *__restrict freq);
+616
查看文件
@@ -0,0 +1,616 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LZFSE_INTERNAL_H
#define LZFSE_INTERNAL_H
// Unlike the tunable parameters defined in lzfse_tunables.h, you probably
// should not modify the values defined in this header. Doing so will either
// break the compressor, or result in a compressed data format that is
// incompatible.
#include "lzfse_fse.h"
#include "lzfse_tunables.h"
#include <limits.h>
#include <stddef.h>
#include <stdint.h>
#if defined(_MSC_VER) && !defined(__clang__)
# define LZFSE_INLINE __forceinline
# define __builtin_expect(X, Y) (X)
# define __attribute__(X)
# pragma warning(disable : 4068) // warning C4068: unknown pragma
#else
# define LZFSE_INLINE static inline __attribute__((__always_inline__))
#endif
// Implement GCC bit scan builtins for MSVC
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
LZFSE_INLINE int __builtin_clz(unsigned int val) {
unsigned long r = 0;
if (_BitScanReverse(&r, val)) {
return 31 - r;
}
return 32;
}
LZFSE_INLINE int __builtin_ctzl(unsigned long val) {
unsigned long r = 0;
if (_BitScanForward(&r, val)) {
return r;
}
return 32;
}
LZFSE_INLINE int __builtin_ctzll(uint64_t val) {
unsigned long r = 0;
#if defined(_M_AMD64) || defined(_M_ARM)
if (_BitScanForward64(&r, val)) {
return r;
}
#else
if (_BitScanForward(&r, (uint32_t)val)) {
return r;
}
if (_BitScanForward(&r, (uint32_t)(val >> 32))) {
return 32 + r;
}
#endif
return 64;
}
#endif
// Throughout LZFSE we refer to "L", "M" and "D"; these will always appear as
// a triplet, and represent a "usual" LZ-style literal and match pair. "L"
// is the number of literal bytes, "M" is the number of match bytes, and "D"
// is the match "distance"; the distance in bytes between the current pointer
// and the start of the match.
#define LZFSE_ENCODE_HASH_VALUES (1 << LZFSE_ENCODE_HASH_BITS)
#define LZFSE_ENCODE_L_SYMBOLS 20
#define LZFSE_ENCODE_M_SYMBOLS 20
#define LZFSE_ENCODE_D_SYMBOLS 64
#define LZFSE_ENCODE_LITERAL_SYMBOLS 256
#define LZFSE_ENCODE_L_STATES 64
#define LZFSE_ENCODE_M_STATES 64
#define LZFSE_ENCODE_D_STATES 256
#define LZFSE_ENCODE_LITERAL_STATES 1024
#define LZFSE_MATCHES_PER_BLOCK 10000
#define LZFSE_LITERALS_PER_BLOCK (4 * LZFSE_MATCHES_PER_BLOCK)
#define LZFSE_DECODE_LITERALS_PER_BLOCK (4 * LZFSE_DECODE_MATCHES_PER_BLOCK)
// LZFSE internal status. These values are used by internal LZFSE routines
// as return codes. There should not be any good reason to change their
// values; it is plausible that additional codes might be added in the
// future.
#define LZFSE_STATUS_OK 0
#define LZFSE_STATUS_SRC_EMPTY -1
#define LZFSE_STATUS_DST_FULL -2
#define LZFSE_STATUS_ERROR -3
// Type representing an offset between elements in a buffer. On 64-bit
// systems, this is stored in a 64-bit container to avoid extra sign-
// extension operations in addressing arithmetic, but the value is always
// representable as a 32-bit signed value in LZFSE's usage.
#if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__)
typedef int64_t lzfse_offset;
#else
typedef int32_t lzfse_offset;
#endif
/*! @abstract History table set. Each line of the history table represents a set
* of candidate match locations, each of which begins with four bytes with the
* same hash. The table contains not only the positions, but also the first
* four bytes at each position. This doubles the memory footprint of the
* table, but allows us to quickly eliminate false-positive matches without
* doing any pointer chasing and without pulling in any additional cachelines.
* This provides a large performance win in practice. */
typedef struct {
int32_t pos[LZFSE_ENCODE_HASH_WIDTH];
uint32_t value[LZFSE_ENCODE_HASH_WIDTH];
} lzfse_history_set;
/*! @abstract An lzfse match is a sequence of bytes in the source buffer that
* exactly matches an earlier (but possibly overlapping) sequence of bytes in
* the same buffer.
* @code
* exeMPLARYexaMPLE
* | | | ||-|--- lzfse_match2.length=3
* | | | ||----- lzfse_match2.pos
* | | |-|------ lzfse_match1.length=3
* | | |-------- lzfse_match1.pos
* | |-------------- lzfse_match2.ref
* |----------------- lzfse_match1.ref
* @endcode
*/
typedef struct {
// Offset of the first byte in the match.
lzfse_offset pos;
// First byte of the source -- the earlier location in the buffer with the
// same contents.
lzfse_offset ref;
// Length of the match.
uint32_t length;
} lzfse_match;
// MARK: - Encoder and Decoder state objects
/*! @abstract Encoder state object. */
typedef struct {
// Pointer to first byte of the source buffer.
const uint8_t *src;
// Length of the source buffer in bytes. Note that this is not a size_t,
// but rather lzfse_offset, which is a signed type. The largest
// representable buffer is 2GB, but arbitrarily large buffers may be
// handled by repeatedly calling the encoder function and "translating"
// the state between calls. When doing this, it is beneficial to use
// blocks smaller than 2GB in order to maintain residency in the last-level
// cache. Consult the implementation of lzfse_encode_buffer for details.
lzfse_offset src_end;
// Offset of the first byte of the next literal to encode in the source
// buffer.
lzfse_offset src_literal;
// Offset of the byte currently being checked for a match.
lzfse_offset src_encode_i;
// The last byte offset to consider for a match. In some uses it makes
// sense to use a smaller offset than src_end.
lzfse_offset src_encode_end;
// Pointer to the next byte to be written in the destination buffer.
uint8_t *dst;
// Pointer to the first byte of the destination buffer.
uint8_t *dst_begin;
// Pointer to one byte past the end of the destination buffer.
uint8_t *dst_end;
// Pending match; will be emitted unless a better match is found.
lzfse_match pending;
// The number of matches written so far. Note that there is no problem in
// using a 32-bit field for this quantity, because the state already limits
// us to at most 2GB of data; there cannot possibly be more matches than
// there are bytes in the input.
uint32_t n_matches;
// The number of literals written so far.
uint32_t n_literals;
// Lengths of found literals.
uint32_t l_values[LZFSE_MATCHES_PER_BLOCK];
// Lengths of found matches.
uint32_t m_values[LZFSE_MATCHES_PER_BLOCK];
// Distances of found matches.
uint32_t d_values[LZFSE_MATCHES_PER_BLOCK];
// Concatenated literal bytes.
uint8_t literals[LZFSE_LITERALS_PER_BLOCK];
// History table used to search for matches. Each entry of the table
// corresponds to a group of four byte sequences in the input stream
// that hash to the same value.
lzfse_history_set history_table[LZFSE_ENCODE_HASH_VALUES];
} lzfse_encoder_state;
/*! @abstract Decoder state object for lzfse compressed blocks. */
typedef struct {
// Number of matches remaining in the block.
uint32_t n_matches;
// Number of bytes used to encode L, M, D triplets for the block.
uint32_t n_lmd_payload_bytes;
// Pointer to the next literal to emit.
const uint8_t *current_literal;
// L, M, D triplet for the match currently being emitted. This is used only
// if we need to restart after reaching the end of the destination buffer in
// the middle of a literal or match.
int32_t l_value, m_value, d_value;
// FSE stream object.
fse_in_stream lmd_in_stream;
// Offset of L,M,D encoding in the input buffer. Because we read through an
// FSE stream *backwards* while decoding, this is decremented as we move
// through a block.
uint32_t lmd_in_buf;
// The current state of the L, M, and D FSE decoders.
uint16_t l_state, m_state, d_state;
// Internal FSE decoder tables for the current block. These have
// alignment forced to 8 bytes to guarantee that a single state's
// entry cannot span two cachelines.
fse_value_decoder_entry l_decoder[LZFSE_ENCODE_L_STATES] __attribute__((__aligned__(8)));
fse_value_decoder_entry m_decoder[LZFSE_ENCODE_M_STATES] __attribute__((__aligned__(8)));
fse_value_decoder_entry d_decoder[LZFSE_ENCODE_D_STATES] __attribute__((__aligned__(8)));
int32_t literal_decoder[LZFSE_ENCODE_LITERAL_STATES];
// The literal stream for the block, plus padding to allow for faster copy
// operations.
uint8_t literals[LZFSE_LITERALS_PER_BLOCK + 64];
} lzfse_compressed_block_decoder_state;
// Decoder state object for uncompressed blocks.
typedef struct { uint32_t n_raw_bytes; } uncompressed_block_decoder_state;
/*! @abstract Decoder state object for lzvn-compressed blocks. */
typedef struct {
uint32_t n_raw_bytes;
uint32_t n_payload_bytes;
uint32_t d_prev;
} lzvn_compressed_block_decoder_state;
/*! @abstract Decoder state object. */
typedef struct {
// Pointer to next byte to read from source buffer (this is advanced as we
// decode; src_begin describe the buffer and do not change).
const uint8_t *src;
// Pointer to first byte of source buffer.
const uint8_t *src_begin;
// Pointer to one byte past the end of the source buffer.
const uint8_t *src_end;
// Pointer to the next byte to write to destination buffer (this is advanced
// as we decode; dst_begin and dst_end describe the buffer and do not change).
uint8_t *dst;
// Pointer to first byte of destination buffer.
uint8_t *dst_begin;
// Pointer to one byte past the end of the destination buffer.
uint8_t *dst_end;
// 1 if we have reached the end of the stream, 0 otherwise.
int end_of_stream;
// magic number of the current block if we are within a block,
// LZFSE_NO_BLOCK_MAGIC otherwise.
uint32_t block_magic;
lzfse_compressed_block_decoder_state compressed_lzfse_block_state;
lzvn_compressed_block_decoder_state compressed_lzvn_block_state;
uncompressed_block_decoder_state uncompressed_block_state;
} lzfse_decoder_state;
// MARK: - Block header objects
#define LZFSE_NO_BLOCK_MAGIC 0x00000000 // 0 (invalid)
#define LZFSE_ENDOFSTREAM_BLOCK_MAGIC 0x24787662 // bvx$ (end of stream)
#define LZFSE_UNCOMPRESSED_BLOCK_MAGIC 0x2d787662 // bvx- (raw data)
#define LZFSE_COMPRESSEDV1_BLOCK_MAGIC 0x31787662 // bvx1 (lzfse compressed, uncompressed tables)
#define LZFSE_COMPRESSEDV2_BLOCK_MAGIC 0x32787662 // bvx2 (lzfse compressed, compressed tables)
#define LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC 0x6e787662 // bvxn (lzvn compressed)
/*! @abstract Uncompressed block header in encoder stream. */
typedef struct {
// Magic number, always LZFSE_UNCOMPRESSED_BLOCK_MAGIC.
uint32_t magic;
// Number of raw bytes in block.
uint32_t n_raw_bytes;
} uncompressed_block_header;
/*! @abstract Compressed block header with uncompressed tables. */
typedef struct {
// Magic number, always LZFSE_COMPRESSEDV1_BLOCK_MAGIC.
uint32_t magic;
// Number of decoded (output) bytes in block.
uint32_t n_raw_bytes;
// Number of encoded (source) bytes in block.
uint32_t n_payload_bytes;
// Number of literal bytes output by block (*not* the number of literals).
uint32_t n_literals;
// Number of matches in block (which is also the number of literals).
uint32_t n_matches;
// Number of bytes used to encode literals.
uint32_t n_literal_payload_bytes;
// Number of bytes used to encode matches.
uint32_t n_lmd_payload_bytes;
// Final encoder states for the block, which will be the initial states for
// the decoder:
// Final accum_nbits for literals stream.
int32_t literal_bits;
// There are four interleaved streams of literals, so there are four final
// states.
uint16_t literal_state[4];
// accum_nbits for the l, m, d stream.
int32_t lmd_bits;
// Final L (literal length) state.
uint16_t l_state;
// Final M (match length) state.
uint16_t m_state;
// Final D (match distance) state.
uint16_t d_state;
// Normalized frequency tables for each stream. Sum of values in each
// array is the number of states.
uint16_t l_freq[LZFSE_ENCODE_L_SYMBOLS];
uint16_t m_freq[LZFSE_ENCODE_M_SYMBOLS];
uint16_t d_freq[LZFSE_ENCODE_D_SYMBOLS];
uint16_t literal_freq[LZFSE_ENCODE_LITERAL_SYMBOLS];
} lzfse_compressed_block_header_v1;
/*! @abstract Compressed block header with compressed tables. Note that because
* freq[] is compressed, the structure-as-stored-in-the-stream is *truncated*;
* we only store the used bytes of freq[]. This means that some extra care must
* be taken when reading one of these headers from the stream. */
typedef struct {
// Magic number, always LZFSE_COMPRESSEDV2_BLOCK_MAGIC.
uint32_t magic;
// Number of decoded (output) bytes in block.
uint32_t n_raw_bytes;
// The fields n_payload_bytes ... d_state from the
// lzfse_compressed_block_header_v1 object are packed into three 64-bit
// fields in the compressed header, as follows:
//
// offset bits value
// 0 20 n_literals
// 20 20 n_literal_payload_bytes
// 40 20 n_matches
// 60 3 literal_bits
// 63 1 --- unused ---
//
// 0 10 literal_state[0]
// 10 10 literal_state[1]
// 20 10 literal_state[2]
// 30 10 literal_state[3]
// 40 20 n_lmd_payload_bytes
// 60 3 lmd_bits
// 63 1 --- unused ---
//
// 0 32 header_size (total header size in bytes; this does not
// correspond to a field in the uncompressed header version,
// but is required; we wouldn't know the size of the
// compresssed header otherwise.
// 32 10 l_state
// 42 10 m_state
// 52 10 d_state
// 62 2 --- unused ---
uint64_t packed_fields[3];
// Variable size freq tables, using a Huffman-style fixed encoding.
// Size allocated here is an upper bound (all values stored on 16 bits).
uint8_t freq[2 * (LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS +
LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS)];
} __attribute__((__packed__, __aligned__(1)))
lzfse_compressed_block_header_v2;
/*! @abstract LZVN compressed block header. */
typedef struct {
// Magic number, always LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC.
uint32_t magic;
// Number of decoded (output) bytes.
uint32_t n_raw_bytes;
// Number of encoded (source) bytes.
uint32_t n_payload_bytes;
} lzvn_compressed_block_header;
// MARK: - LZFSE encode/decode interfaces
int lzfse_encode_init(lzfse_encoder_state *s);
int lzfse_encode_translate(lzfse_encoder_state *s, lzfse_offset delta);
int lzfse_encode_base(lzfse_encoder_state *s);
int lzfse_encode_finish(lzfse_encoder_state *s);
int lzfse_decode(lzfse_decoder_state *s);
// MARK: - LZVN encode/decode interfaces
// Minimum source buffer size for compression. Smaller buffers will not be
// compressed; the lzvn encoder will simply return.
#define LZVN_ENCODE_MIN_SRC_SIZE ((size_t)8)
// Maximum source buffer size for compression. Larger buffers will be
// compressed partially.
#define LZVN_ENCODE_MAX_SRC_SIZE ((size_t)0xffffffffU)
// Minimum destination buffer size for compression. No compression will take
// place if smaller.
#define LZVN_ENCODE_MIN_DST_SIZE ((size_t)8)
size_t lzvn_decode_scratch_size(void);
size_t lzvn_encode_scratch_size(void);
size_t lzvn_encode_buffer(void *__restrict dst, size_t dst_size,
const void *__restrict src, size_t src_size,
void *__restrict work);
size_t lzvn_decode_buffer(void *__restrict dst, size_t dst_size,
const void *__restrict src, size_t src_size);
/*! @abstract Signed offset in buffers, stored on either 32 or 64 bits. */
#if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__)
typedef int64_t lzvn_offset;
#else
typedef int32_t lzvn_offset;
#endif
// MARK: - LZFSE utility functions
/*! @abstract Load bytes from memory location SRC. */
LZFSE_INLINE uint16_t load2(const void *ptr) {
uint16_t data;
memcpy(&data, ptr, sizeof data);
return data;
}
LZFSE_INLINE uint32_t load4(const void *ptr) {
uint32_t data;
memcpy(&data, ptr, sizeof data);
return data;
}
LZFSE_INLINE uint64_t load8(const void *ptr) {
uint64_t data;
memcpy(&data, ptr, sizeof data);
return data;
}
/*! @abstract Store bytes to memory location DST. */
LZFSE_INLINE void store2(void *ptr, uint16_t data) {
memcpy(ptr, &data, sizeof data);
}
LZFSE_INLINE void store4(void *ptr, uint32_t data) {
memcpy(ptr, &data, sizeof data);
}
LZFSE_INLINE void store8(void *ptr, uint64_t data) {
memcpy(ptr, &data, sizeof data);
}
/*! @abstract Load+store bytes from locations SRC to DST. Not intended for use
* with overlapping buffers. Note that for LZ-style compression, you need
* copies to behave like naive memcpy( ) implementations do, splatting the
* leading sequence if the buffers overlap. This copy does not do that, so
* should not be used with overlapping buffers. */
LZFSE_INLINE void copy8(void *dst, const void *src) { store8(dst, load8(src)); }
LZFSE_INLINE void copy16(void *dst, const void *src) {
uint64_t m0 = load8(src);
uint64_t m1 = load8((const unsigned char *)src + 8);
store8(dst, m0);
store8((unsigned char *)dst + 8, m1);
}
// ===============================================================
// Bitfield Operations
/*! @abstract Extracts \p width bits from \p container, starting with \p lsb; if
* we view \p container as a bit array, we extract \c container[lsb:lsb+width]. */
LZFSE_INLINE uintmax_t extract(uintmax_t container, unsigned lsb,
unsigned width) {
static const size_t container_width = sizeof container * 8;
assert(lsb < container_width);
assert(width > 0 && width <= container_width);
assert(lsb + width <= container_width);
if (width == container_width)
return container;
return (container >> lsb) & (((uintmax_t)1 << width) - 1);
}
/*! @abstract Inserts \p width bits from \p data into \p container, starting with \p lsb.
* Viewed as bit arrays, the operations is:
* @code
* container[:lsb] is unchanged
* container[lsb:lsb+width] <-- data[0:width]
* container[lsb+width:] is unchanged
* @endcode
*/
LZFSE_INLINE uintmax_t insert(uintmax_t container, uintmax_t data, unsigned lsb,
unsigned width) {
static const size_t container_width = sizeof container * 8;
assert(lsb < container_width);
assert(width > 0 && width <= container_width);
assert(lsb + width <= container_width);
if (width == container_width)
return container;
uintmax_t mask = ((uintmax_t)1 << width) - 1;
return (container & ~(mask << lsb)) | (data & mask) << lsb;
}
/*! @abstract Perform sanity checks on the values of lzfse_compressed_block_header_v1.
* Test that the field values are in the allowed limits, verify that the
* frequency tables sum to value less than total number of states.
* @return 0 if all tests passed.
* @return negative error code with 1 bit set for each failed test. */
LZFSE_INLINE int lzfse_check_block_header_v1(
const lzfse_compressed_block_header_v1 *header) {
int tests_results = 0;
tests_results =
tests_results |
((header->magic == LZFSE_COMPRESSEDV1_BLOCK_MAGIC) ? 0 : (1 << 0));
tests_results =
tests_results |
((header->n_literals <= LZFSE_LITERALS_PER_BLOCK) ? 0 : (1 << 1));
tests_results =
tests_results |
((header->n_matches <= LZFSE_MATCHES_PER_BLOCK) ? 0 : (1 << 2));
uint16_t literal_state[4];
memcpy(literal_state, header->literal_state, sizeof(uint16_t) * 4);
tests_results =
tests_results |
((literal_state[0] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 3));
tests_results =
tests_results |
((literal_state[1] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 4));
tests_results =
tests_results |
((literal_state[2] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 5));
tests_results =
tests_results |
((literal_state[3] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 6));
tests_results = tests_results |
((header->l_state < LZFSE_ENCODE_L_STATES) ? 0 : (1 << 7));
tests_results = tests_results |
((header->m_state < LZFSE_ENCODE_M_STATES) ? 0 : (1 << 8));
tests_results = tests_results |
((header->d_state < LZFSE_ENCODE_D_STATES) ? 0 : (1 << 9));
int res;
res = fse_check_freq(header->l_freq, LZFSE_ENCODE_L_SYMBOLS,
LZFSE_ENCODE_L_STATES);
tests_results = tests_results | ((res == 0) ? 0 : (1 << 10));
res = fse_check_freq(header->m_freq, LZFSE_ENCODE_M_SYMBOLS,
LZFSE_ENCODE_M_STATES);
tests_results = tests_results | ((res == 0) ? 0 : (1 << 11));
res = fse_check_freq(header->d_freq, LZFSE_ENCODE_D_SYMBOLS,
LZFSE_ENCODE_D_STATES);
tests_results = tests_results | ((res == 0) ? 0 : (1 << 12));
res = fse_check_freq(header->literal_freq, LZFSE_ENCODE_LITERAL_SYMBOLS,
LZFSE_ENCODE_LITERAL_STATES);
tests_results = tests_results | ((res == 0) ? 0 : (1 << 13));
if (tests_results) {
return tests_results | 0x80000000; // each 1 bit is a test that failed
// (except for the sign bit)
}
return 0; // OK
}
// MARK: - L, M, D encoding constants for LZFSE
// Largest encodable L (literal length), M (match length) and D (match
// distance) values.
#define LZFSE_ENCODE_MAX_L_VALUE 315
#define LZFSE_ENCODE_MAX_M_VALUE 2359
#define LZFSE_ENCODE_MAX_D_VALUE 262139
/*! @abstract The L, M, D data streams are all encoded as a "base" value, which is
* FSE-encoded, and an "extra bits" value, which is the difference between
* value and base, and is simply represented as a raw bit value (because it
* is the low-order bits of a larger number, not much entropy can be
* extracted from these bits by more complex encoding schemes). The following
* tables represent the number of low-order bits to encode separately and the
* base values for each of L, M, and D.
*
* @note The inverse tables for mapping the other way are significantly larger.
* Those tables have been split out to lzfse_encode_tables.h in order to keep
* this file relatively small. */
static const uint8_t l_extra_bits[LZFSE_ENCODE_L_SYMBOLS] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 5, 8
};
static const int32_t l_base_value[LZFSE_ENCODE_L_SYMBOLS] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 28, 60
};
static const uint8_t m_extra_bits[LZFSE_ENCODE_M_SYMBOLS] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 8, 11
};
static const int32_t m_base_value[LZFSE_ENCODE_M_SYMBOLS] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 24, 56, 312
};
static const uint8_t d_extra_bits[LZFSE_ENCODE_D_SYMBOLS] = {
0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,
12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15
};
static const int32_t d_base_value[LZFSE_ENCODE_D_SYMBOLS] = {
0, 1, 2, 3, 4, 6, 8, 10, 12, 16,
20, 24, 28, 36, 44, 52, 60, 76, 92, 108,
124, 156, 188, 220, 252, 316, 380, 444, 508, 636,
764, 892, 1020, 1276, 1532, 1788, 2044, 2556, 3068, 3580,
4092, 5116, 6140, 7164, 8188, 10236, 12284, 14332, 16380, 20476,
24572, 28668, 32764, 40956, 49148, 57340, 65532, 81916, 98300, 114684,
131068, 163836, 196604, 229372
};
#endif // LZFSE_INTERNAL_H
+336
查看文件
@@ -0,0 +1,336 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// LZFSE command line tool
#if !defined(_POSIX_C_SOURCE) || (_POSIX_C_SOURCE < 200112L)
# undef _POSIX_C_SOURCE
# define _POSIX_C_SOURCE 200112L
#endif
#if defined(_MSC_VER)
# if !defined(_CRT_NONSTDC_NO_DEPRECATE)
# define _CRT_NONSTDC_NO_DEPRECATE
# endif
# if !defined(_CRT_SECURE_NO_WARNINGS)
# define _CRT_SECURE_NO_WARNINGS
# endif
# if !defined(__clang__)
# define inline __inline
# endif
#endif
#include "lzfse.h"
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#if defined(_MSC_VER)
# include <io.h>
# include <windows.h>
#else
# include <sys/time.h>
# include <unistd.h>
#endif
// Same as realloc(x,s), except x is freed when realloc fails
static inline void *lzfse_reallocf(void *x, size_t s) {
void *y = realloc(x, s);
if (y == 0) {
free(x);
return 0;
}
return y;
}
static double get_time() {
#if defined(_MSC_VER)
LARGE_INTEGER count, freq;
if (QueryPerformanceFrequency(&freq) && QueryPerformanceCounter(&count)) {
return (double)count.QuadPart / (double)freq.QuadPart;
}
return 1.0e-3 * (double)GetTickCount();
#else
struct timeval tv;
if (gettimeofday(&tv, 0) != 0) {
perror("gettimeofday");
exit(1);
}
return (double)tv.tv_sec + 1.0e-6 * (double)tv.tv_usec;
#endif
}
//--------------------
enum { LZFSE_ENCODE = 0, LZFSE_DECODE };
void usage(int argc, char **argv) {
fprintf(
stderr,
"Usage: %s -encode|-decode [-i input_file] [-o output_file] [-h] [-v]\n",
argv[0]);
}
#define USAGE(argc, argv) \
do { \
usage(argc, argv); \
exit(0); \
} while (0)
#define USAGE_MSG(argc, argv, ...) \
do { \
usage(argc, argv); \
fprintf(stderr, __VA_ARGS__); \
exit(1); \
} while (0)
int main(int argc, char **argv) {
const char *in_file = 0; // stdin
const char *out_file = 0; // stdout
int op = -1; // invalid op
int verbosity = 0; // quiet
// Parse options
for (int i = 1; i < argc;) {
// no args
const char *a = argv[i++];
if (strcmp(a, "-h") == 0)
USAGE(argc, argv);
if (strcmp(a, "-v") == 0) {
verbosity++;
continue;
}
if (strcmp(a, "-encode") == 0) {
op = LZFSE_ENCODE;
continue;
}
if (strcmp(a, "-decode") == 0) {
op = LZFSE_DECODE;
continue;
}
// one arg
const char **arg_var = 0;
if (strcmp(a, "-i") == 0 && in_file == 0)
arg_var = &in_file;
else if (strcmp(a, "-o") == 0 && out_file == 0)
arg_var = &out_file;
if (arg_var != 0) {
// Flag is recognized. Check if there is an argument.
if (i == argc)
USAGE_MSG(argc, argv, "Error: Missing arg after %s\n", a);
*arg_var = argv[i++];
continue;
}
USAGE_MSG(argc, argv, "Error: invalid flag %s\n", a);
}
if (op < 0)
USAGE_MSG(argc, argv, "Error: -encode|-decode required\n");
// Info
if (verbosity > 0) {
if (op == LZFSE_ENCODE)
fprintf(stderr, "LZFSE encode\n");
if (op == LZFSE_DECODE)
fprintf(stderr, "LZFSE decode\n");
fprintf(stderr, "Input: %s\n", in_file ? in_file : "stdin");
fprintf(stderr, "Output: %s\n", out_file ? out_file : "stdout");
}
// Load input
size_t in_allocated = 0; // allocated in IN
size_t in_size = 0; // used in IN
uint8_t *in = 0; // input buffer
int in_fd = -1; // input file desc
if (in_file != 0) {
// If we have a file name, open it, and allocate the exact input size
struct stat st;
#if defined(_WIN32)
in_fd = open(in_file, O_RDONLY | O_BINARY);
#else
in_fd = open(in_file, O_RDONLY);
#endif
if (in_fd < 0) {
perror(in_file);
exit(1);
}
if (fstat(in_fd, &st) != 0) {
perror(in_file);
exit(1);
}
if (st.st_size > SIZE_MAX) {
fprintf(stderr, "File is too large\n");
exit(1);
}
in_allocated = (size_t)st.st_size;
} else {
// Otherwise, read from stdin, and allocate to 1 MB, grow as needed
in_allocated = 1 << 20;
in_fd = 0;
#if defined(_WIN32)
if (setmode(in_fd, O_BINARY) == -1) {
perror("setmode");
exit(1);
}
#endif
}
in = (uint8_t *)malloc(in_allocated);
if (in == 0) {
perror("malloc");
exit(1);
}
while (1) {
// re-alloc if needed
if (in_size == in_allocated) {
if (in_allocated < (100 << 20))
in_allocated <<= 1; // double it
else
in_allocated += (100 << 20); // or add 100 MB if already large
in = lzfse_reallocf(in, in_allocated);
if (in == 0) {
perror("malloc");
exit(1);
}
}
ptrdiff_t r = read(in_fd, in + in_size, in_allocated - in_size);
if (r < 0) {
perror("read");
exit(1);
}
if (r == 0)
break; // end of file
in_size += (size_t)r;
}
if (in_file != 0) {
close(in_fd);
in_fd = -1;
}
// Size info
if (verbosity > 0) {
fprintf(stderr, "Input size: %zu B\n", in_size);
}
// Encode/decode
// Compute size for result buffer; we assume here that encode shrinks size,
// and that decode grows by no more than 4x. These are reasonable common-
// case guidelines, but are not formally guaranteed to be satisfied.
size_t out_allocated = (op == LZFSE_ENCODE) ? in_size : (4 * in_size);
size_t out_size = 0;
size_t aux_allocated = (op == LZFSE_ENCODE) ? lzfse_encode_scratch_size()
: lzfse_decode_scratch_size();
void *aux = aux_allocated ? malloc(aux_allocated) : 0;
if (aux_allocated != 0 && aux == 0) {
perror("malloc");
exit(1);
}
uint8_t *out = (uint8_t *)malloc(out_allocated);
if (out == 0) {
perror("malloc");
exit(1);
}
double c0 = get_time();
while (1) {
if (op == LZFSE_ENCODE)
out_size = lzfse_encode_buffer(out, out_allocated, in, in_size, aux);
else
out_size = lzfse_decode_buffer(out, out_allocated, in, in_size, aux);
// If output buffer was too small, grow and retry.
if (out_size == 0 || (op == LZFSE_DECODE && out_size == out_allocated)) {
if (verbosity > 0)
fprintf(stderr, "Output buffer was too small, increasing size...\n");
out_allocated <<= 1;
out = (uint8_t *)lzfse_reallocf(out, out_allocated);
if (out == 0) {
perror("malloc");
exit(1);
}
continue;
}
break;
}
double c1 = get_time();
if (verbosity > 0) {
fprintf(stderr, "Output size: %zu B\n", out_size);
size_t raw_size = (op == LZFSE_ENCODE) ? in_size : out_size;
size_t compressed_size = (op == LZFSE_ENCODE) ? out_size : in_size;
fprintf(stderr, "Compression ratio: %.3f\n",
(double)raw_size / (double)compressed_size);
double ns_per_byte = 1.0e9 * (c1 - c0) / (double)raw_size;
double mb_per_s = (double)raw_size / 1024.0 / 1024.0 / (c1 - c0);
fprintf(stderr, "Speed: %.2f ns/B, %.2f MB/s\n",ns_per_byte,mb_per_s);
}
// Write output
int out_fd = -1;
if (out_file) {
#if defined(_WIN32)
out_fd = open(out_file, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY,
S_IWRITE);
#else
out_fd = open(out_file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
#endif
if (out_fd < 0) {
perror(out_file);
exit(1);
}
} else {
out_fd = 1; // stdout
#if defined(_WIN32)
if (setmode(out_fd, O_BINARY) == -1) {
perror("setmode");
exit(1);
}
#endif
}
for (size_t out_pos = 0; out_pos < out_size;) {
ptrdiff_t w = write(out_fd, out + out_pos, out_size - out_pos);
if (w < 0) {
perror("write");
exit(1);
}
if (w == 0) {
fprintf(stderr, "Failed to write to output file\n");
exit(1);
}
out_pos += (size_t)w;
}
if (out_file != 0) {
close(out_fd);
out_fd = -1;
}
free(in);
free(out);
free(aux);
return 0; // OK
}
+60
查看文件
@@ -0,0 +1,60 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LZFSE_TUNABLES_H
#define LZFSE_TUNABLES_H
// Parameters controlling details of the LZ-style match search. These values
// may be modified to fine tune compression ratio vs. encoding speed, while
// keeping the compressed format compatible with LZFSE. Note that
// modifying them will also change the amount of work space required by
// the encoder. The values here are those used in the compression library
// on iOS and OS X.
// Number of bits for hash function to produce. Should be in the range
// [10, 16]. Larger values reduce the number of false-positive found during
// the match search, and expand the history table, which may allow additional
// matches to be found, generally improving the achieved compression ratio.
// Larger values also increase the workspace size, and make it less likely
// that the history table will be present in cache, which reduces performance.
#define LZFSE_ENCODE_HASH_BITS 14
// Number of positions to store for each line in the history table. May
// be either 4 or 8. Using 8 doubles the size of the history table, which
// increases the chance of finding matches (thus improving compression ratio),
// but also increases the workspace size.
#define LZFSE_ENCODE_HASH_WIDTH 4
// Match length in bytes to cause immediate emission. Generally speaking,
// LZFSE maintains multiple candidate matches and waits to decide which match
// to emit until more information is available. When a match exceeds this
// threshold, it is emitted immediately. Thus, smaller values may give
// somewhat better performance, and larger values may give somewhat better
// compression ratios.
#define LZFSE_ENCODE_GOOD_MATCH 40
// When the source buffer is very small, LZFSE doesn't compress as well as
// some simpler algorithms. To maintain reasonable compression for these
// cases, we transition to use LZVN instead if the size of the source buffer
// is below this threshold.
#define LZFSE_ENCODE_LZVN_THRESHOLD 4096
#endif // LZFSE_TUNABLES_H
+711
查看文件
@@ -0,0 +1,711 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// LZVN low-level decoder
#include "lzvn_decode_base.h"
#if !defined(HAVE_LABELS_AS_VALUES)
# if defined(__GNUC__) || defined(__clang__)
# define HAVE_LABELS_AS_VALUES 1
# else
# define HAVE_LABELS_AS_VALUES 0
# endif
#endif
// Both the source and destination buffers are represented by a pointer and
// a length; they are *always* updated in concert using this macro; however
// many bytes the pointer is advanced, the length is decremented by the same
// amount. Thus, pointer + length always points to the byte one past the end
// of the buffer.
#define PTR_LEN_INC(_pointer, _length, _increment) \
(_pointer += _increment, _length -= _increment)
// Update state with current positions and distance, corresponding to the
// beginning of an instruction in both streams
#define UPDATE_GOOD \
(state->src = src_ptr, state->dst = dst_ptr, state->d_prev = D)
void lzvn_decode(lzvn_decoder_state *state) {
#if HAVE_LABELS_AS_VALUES
// Jump table for all instructions
static const void *opc_tbl[256] = {
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&eos, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&nop, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&nop, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef,
&&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d,
&&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d,
&&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d,
&&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d,
&&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef,
&&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef,
&&lrg_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l,
&&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l,
&&lrg_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m,
&&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m};
#endif
size_t src_len = state->src_end - state->src;
size_t dst_len = state->dst_end - state->dst;
if (src_len == 0 || dst_len == 0)
return; // empty buffer
const unsigned char *src_ptr = state->src;
unsigned char *dst_ptr = state->dst;
size_t D = state->d_prev;
size_t M;
size_t L;
size_t opc_len;
// Do we have a partially expanded match saved in state?
if (state->L != 0 || state->M != 0) {
L = state->L;
M = state->M;
D = state->D;
opc_len = 0; // we already skipped the op
state->L = state->M = state->D = 0;
if (M == 0)
goto copy_literal;
if (L == 0)
goto copy_match;
goto copy_literal_and_match;
}
unsigned char opc = src_ptr[0];
#if HAVE_LABELS_AS_VALUES
goto *opc_tbl[opc];
#else
for (;;) {
switch (opc) {
#endif
// ===============================================================
// These four opcodes (sml_d, med_d, lrg_d, and pre_d) encode both a
// literal and a match. The bulk of their implementations are shared;
// each label here only does the work of setting the opcode length (not
// including any literal bytes), and extracting the literal length, match
// length, and match distance (except in pre_d). They then jump into the
// shared implementation to actually output the literal and match bytes.
//
// No error checking happens in the first stage, except for ensuring that
// the source has enough length to represent the full opcode before
// reading past the first byte.
sml_d:
#if !HAVE_LABELS_AS_VALUES
case 0:
case 1:
case 2:
case 3:
case 4:
case 5:
case 8:
case 9:
case 10:
case 11:
case 12:
case 13:
case 16:
case 17:
case 18:
case 19:
case 20:
case 21:
case 24:
case 25:
case 26:
case 27:
case 28:
case 29:
case 32:
case 33:
case 34:
case 35:
case 36:
case 37:
case 40:
case 41:
case 42:
case 43:
case 44:
case 45:
case 48:
case 49:
case 50:
case 51:
case 52:
case 53:
case 56:
case 57:
case 58:
case 59:
case 60:
case 61:
case 64:
case 65:
case 66:
case 67:
case 68:
case 69:
case 72:
case 73:
case 74:
case 75:
case 76:
case 77:
case 80:
case 81:
case 82:
case 83:
case 84:
case 85:
case 88:
case 89:
case 90:
case 91:
case 92:
case 93:
case 96:
case 97:
case 98:
case 99:
case 100:
case 101:
case 104:
case 105:
case 106:
case 107:
case 108:
case 109:
case 128:
case 129:
case 130:
case 131:
case 132:
case 133:
case 136:
case 137:
case 138:
case 139:
case 140:
case 141:
case 144:
case 145:
case 146:
case 147:
case 148:
case 149:
case 152:
case 153:
case 154:
case 155:
case 156:
case 157:
case 192:
case 193:
case 194:
case 195:
case 196:
case 197:
case 200:
case 201:
case 202:
case 203:
case 204:
case 205:
#endif
UPDATE_GOOD;
// "small distance": This opcode has the structure LLMMMDDD DDDDDDDD LITERAL
// where the length of literal (0-3 bytes) is encoded by the high 2 bits of
// the first byte. We first extract the literal length so we know how long
// the opcode is, then check that the source can hold both this opcode and
// at least one byte of the next (because any valid input stream must be
// terminated with an eos token).
opc_len = 2;
L = (size_t)extract(opc, 6, 2);
M = (size_t)extract(opc, 3, 3) + 3;
// We need to ensure that the source buffer is long enough that we can
// safely read this entire opcode, the literal that follows, and the first
// byte of the next opcode. Once we satisfy this requirement, we can
// safely unpack the match distance. A check similar to this one is
// present in all the opcode implementations.
if (src_len <= opc_len + L)
return; // source truncated
D = (size_t)extract(opc, 0, 3) << 8 | src_ptr[1];
goto copy_literal_and_match;
med_d:
#if !HAVE_LABELS_AS_VALUES
case 160:
case 161:
case 162:
case 163:
case 164:
case 165:
case 166:
case 167:
case 168:
case 169:
case 170:
case 171:
case 172:
case 173:
case 174:
case 175:
case 176:
case 177:
case 178:
case 179:
case 180:
case 181:
case 182:
case 183:
case 184:
case 185:
case 186:
case 187:
case 188:
case 189:
case 190:
case 191:
#endif
UPDATE_GOOD;
// "medium distance": This is a minor variant of the "small distance"
// encoding, where we will now use two extra bytes instead of one to encode
// the restof the match length and distance. This allows an extra two bits
// for the match length, and an extra three bits for the match distance. The
// full structure of the opcode is 101LLMMM DDDDDDMM DDDDDDDD LITERAL.
opc_len = 3;
L = (size_t)extract(opc, 3, 2);
if (src_len <= opc_len + L)
return; // source truncated
uint16_t opc23 = load2(&src_ptr[1]);
M = (size_t)((extract(opc, 0, 3) << 2 | extract(opc23, 0, 2)) + 3);
D = (size_t)extract(opc23, 2, 14);
goto copy_literal_and_match;
lrg_d:
#if !HAVE_LABELS_AS_VALUES
case 7:
case 15:
case 23:
case 31:
case 39:
case 47:
case 55:
case 63:
case 71:
case 79:
case 87:
case 95:
case 103:
case 111:
case 135:
case 143:
case 151:
case 159:
case 199:
case 207:
#endif
UPDATE_GOOD;
// "large distance": This is another variant of the "small distance"
// encoding, where we will now use two extra bytes to encode the match
// distance, which allows distances up to 65535 to be represented. The full
// structure of the opcode is LLMMM111 DDDDDDDD DDDDDDDD LITERAL.
opc_len = 3;
L = (size_t)extract(opc, 6, 2);
M = (size_t)extract(opc, 3, 3) + 3;
if (src_len <= opc_len + L)
return; // source truncated
D = load2(&src_ptr[1]);
goto copy_literal_and_match;
pre_d:
#if !HAVE_LABELS_AS_VALUES
case 70:
case 78:
case 86:
case 94:
case 102:
case 110:
case 134:
case 142:
case 150:
case 158:
case 198:
case 206:
#endif
UPDATE_GOOD;
// "previous distance": This opcode has the structure LLMMM110, where the
// length of the literal (0-3 bytes) is encoded by the high 2 bits of the
// first byte. We first extract the literal length so we know how long
// the opcode is, then check that the source can hold both this opcode and
// at least one byte of the next (because any valid input stream must be
// terminated with an eos token).
opc_len = 1;
L = (size_t)extract(opc, 6, 2);
M = (size_t)extract(opc, 3, 3) + 3;
if (src_len <= opc_len + L)
return; // source truncated
goto copy_literal_and_match;
copy_literal_and_match:
// Common implementation of writing data for opcodes that have both a
// literal and a match. We begin by advancing the source pointer past
// the opcode, so that it points at the first literal byte (if L
// is non-zero; otherwise it points at the next opcode).
PTR_LEN_INC(src_ptr, src_len, opc_len);
// Now we copy the literal from the source pointer to the destination.
if (__builtin_expect(dst_len >= 4 && src_len >= 4, 1)) {
// The literal is 0-3 bytes; if we are not near the end of the buffer,
// we can safely just do a 4 byte copy (which is guaranteed to cover
// the complete literal, and may include some other bytes as well).
store4(dst_ptr, load4(src_ptr));
} else if (L <= dst_len) {
// We are too close to the end of either the input or output stream
// to be able to safely use a four-byte copy, but we will not exhaust
// either stream (we already know that the source will not be
// exhausted from checks in the individual opcode implementations,
// and we just tested that dst_len > L). Thus, we need to do a
// byte-by-byte copy of the literal. This is slow, but it can only ever
// happen near the very end of a buffer, so it is not an important case to
// optimize.
for (size_t i = 0; i < L; ++i)
dst_ptr[i] = src_ptr[i];
} else {
// Destination truncated: fill DST, and store partial match
// Copy partial literal
for (size_t i = 0; i < dst_len; ++i)
dst_ptr[i] = src_ptr[i];
// Save state
state->src = src_ptr + dst_len;
state->dst = dst_ptr + dst_len;
state->L = L - dst_len;
state->M = M;
state->D = D;
return; // destination truncated
}
// Having completed the copy of the literal, we advance both the source
// and destination pointers by the number of literal bytes.
PTR_LEN_INC(dst_ptr, dst_len, L);
PTR_LEN_INC(src_ptr, src_len, L);
// Check if the match distance is valid; matches must not reference
// bytes that preceed the start of the output buffer, nor can the match
// distance be zero.
if (D > dst_ptr - state->dst_begin || D == 0)
goto invalid_match_distance;
copy_match:
// Now we copy the match from dst_ptr - D to dst_ptr. It is important to keep
// in mind that we may have D < M, in which case the source and destination
// windows overlap in the copy. The semantics of the match copy are *not*
// those of memmove( ); if the buffers overlap it needs to behave as though
// we were copying byte-by-byte in increasing address order. If, for example,
// D is 1, the copy operation is equivalent to:
//
// memset(dst_ptr, dst_ptr[-1], M);
//
// i.e. it splats the previous byte. This means that we need to be very
// careful about using wide loads or stores to perform the copy operation.
if (__builtin_expect(dst_len >= M + 7 && D >= 8, 1)) {
// We are not near the end of the buffer, and the match distance
// is at least eight. Thus, we can safely loop using eight byte
// copies. The last of these may slop over the intended end of
// the match, but this is OK because we know we have a safety bound
// away from the end of the destination buffer.
for (size_t i = 0; i < M; i += 8)
store8(&dst_ptr[i], load8(&dst_ptr[i - D]));
} else if (M <= dst_len) {
// Either the match distance is too small, or we are too close to
// the end of the buffer to safely use eight byte copies. Fall back
// on a simple byte-by-byte implementation.
for (size_t i = 0; i < M; ++i)
dst_ptr[i] = dst_ptr[i - D];
} else {
// Destination truncated: fill DST, and store partial match
// Copy partial match
for (size_t i = 0; i < dst_len; ++i)
dst_ptr[i] = dst_ptr[i - D];
// Save state
state->src = src_ptr;
state->dst = dst_ptr + dst_len;
state->L = 0;
state->M = M - dst_len;
state->D = D;
return; // destination truncated
}
// Update the destination pointer and length to account for the bytes
// written by the match, then load the next opcode byte and branch to
// the appropriate implementation.
PTR_LEN_INC(dst_ptr, dst_len, M);
opc = src_ptr[0];
#if HAVE_LABELS_AS_VALUES
goto *opc_tbl[opc];
#else
break;
#endif
// ===============================================================
// Opcodes representing only a match (no literal).
// These two opcodes (lrg_m and sml_m) encode only a match. The match
// distance is carried over from the previous opcode, so all they need
// to encode is the match length. We are able to reuse the match copy
// sequence from the literal and match opcodes to perform the actual
// copy implementation.
sml_m:
#if !HAVE_LABELS_AS_VALUES
case 241:
case 242:
case 243:
case 244:
case 245:
case 246:
case 247:
case 248:
case 249:
case 250:
case 251:
case 252:
case 253:
case 254:
case 255:
#endif
UPDATE_GOOD;
// "small match": This opcode has no literal, and uses the previous match
// distance (i.e. it encodes only the match length), in a single byte as
// 1111MMMM.
opc_len = 1;
if (src_len <= opc_len)
return; // source truncated
M = (size_t)extract(opc, 0, 4);
PTR_LEN_INC(src_ptr, src_len, opc_len);
goto copy_match;
lrg_m:
#if !HAVE_LABELS_AS_VALUES
case 240:
#endif
UPDATE_GOOD;
// "large match": This opcode has no literal, and uses the previous match
// distance (i.e. it encodes only the match length). It is encoded in two
// bytes as 11110000 MMMMMMMM. Because matches smaller than 16 bytes can
// be represented by sml_m, there is an implicit bias of 16 on the match
// length; the representable values are [16,271].
opc_len = 2;
if (src_len <= opc_len)
return; // source truncated
M = src_ptr[1] + 16;
PTR_LEN_INC(src_ptr, src_len, opc_len);
goto copy_match;
// ===============================================================
// Opcodes representing only a literal (no match).
// These two opcodes (lrg_l and sml_l) encode only a literal. There is no
// match length or match distance to worry about (but we need to *not*
// touch D, as it must be preserved between opcodes).
sml_l:
#if !HAVE_LABELS_AS_VALUES
case 225:
case 226:
case 227:
case 228:
case 229:
case 230:
case 231:
case 232:
case 233:
case 234:
case 235:
case 236:
case 237:
case 238:
case 239:
#endif
UPDATE_GOOD;
// "small literal": This opcode has no match, and encodes only a literal
// of length up to 15 bytes. The format is 1110LLLL LITERAL.
opc_len = 1;
L = (size_t)extract(opc, 0, 4);
goto copy_literal;
lrg_l:
#if !HAVE_LABELS_AS_VALUES
case 224:
#endif
UPDATE_GOOD;
// "large literal": This opcode has no match, and uses the previous match
// distance (i.e. it encodes only the match length). It is encoded in two
// bytes as 11100000 LLLLLLLL LITERAL. Because literals smaller than 16
// bytes can be represented by sml_l, there is an implicit bias of 16 on
// the literal length; the representable values are [16,271].
opc_len = 2;
if (src_len <= 2)
return; // source truncated
L = src_ptr[1] + 16;
goto copy_literal;
copy_literal:
// Check that the source buffer is large enough to hold the complete
// literal and at least the first byte of the next opcode. If so, advance
// the source pointer to point to the first byte of the literal and adjust
// the source length accordingly.
if (src_len <= opc_len + L)
return; // source truncated
PTR_LEN_INC(src_ptr, src_len, opc_len);
// Now we copy the literal from the source pointer to the destination.
if (dst_len >= L + 7 && src_len >= L + 7) {
// We are not near the end of the source or destination buffers; thus
// we can safely copy the literal using wide copies, without worrying
// about reading or writing past the end of either buffer.
for (size_t i = 0; i < L; i += 8)
store8(&dst_ptr[i], load8(&src_ptr[i]));
} else if (L <= dst_len) {
// We are too close to the end of either the input or output stream
// to be able to safely use an eight-byte copy. Instead we copy the
// literal byte-by-byte.
for (size_t i = 0; i < L; ++i)
dst_ptr[i] = src_ptr[i];
} else {
// Destination truncated: fill DST, and store partial match
// Copy partial literal
for (size_t i = 0; i < dst_len; ++i)
dst_ptr[i] = src_ptr[i];
// Save state
state->src = src_ptr + dst_len;
state->dst = dst_ptr + dst_len;
state->L = L - dst_len;
state->M = 0;
state->D = D;
return; // destination truncated
}
// Having completed the copy of the literal, we advance both the source
// and destination pointers by the number of literal bytes.
PTR_LEN_INC(dst_ptr, dst_len, L);
PTR_LEN_INC(src_ptr, src_len, L);
// Load the first byte of the next opcode, and jump to its implementation.
opc = src_ptr[0];
#if HAVE_LABELS_AS_VALUES
goto *opc_tbl[opc];
#else
break;
#endif
// ===============================================================
// Other opcodes
nop:
#if !HAVE_LABELS_AS_VALUES
case 14:
case 22:
#endif
UPDATE_GOOD;
opc_len = 1;
if (src_len <= opc_len)
return; // source truncated
PTR_LEN_INC(src_ptr, src_len, opc_len);
opc = src_ptr[0];
#if HAVE_LABELS_AS_VALUES
goto *opc_tbl[opc];
#else
break;
#endif
eos:
#if !HAVE_LABELS_AS_VALUES
case 6:
#endif
opc_len = 8;
if (src_len < opc_len)
return; // source truncated (here we don't need an extra byte for next op
// code)
PTR_LEN_INC(src_ptr, src_len, opc_len);
state->end_of_stream = 1;
UPDATE_GOOD;
return; // end-of-stream
// ===============================================================
// Return on error
udef:
#if !HAVE_LABELS_AS_VALUES
case 30:
case 38:
case 46:
case 54:
case 62:
case 112:
case 113:
case 114:
case 115:
case 116:
case 117:
case 118:
case 119:
case 120:
case 121:
case 122:
case 123:
case 124:
case 125:
case 126:
case 127:
case 208:
case 209:
case 210:
case 211:
case 212:
case 213:
case 214:
case 215:
case 216:
case 217:
case 218:
case 219:
case 220:
case 221:
case 222:
case 223:
#endif
invalid_match_distance:
return; // we already updated state
#if !HAVE_LABELS_AS_VALUES
}
}
#endif
}
+68
查看文件
@@ -0,0 +1,68 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// LZVN low-level decoder (v2)
// Functions in the low-level API should switch to these at some point.
// Apr 2014
#ifndef LZVN_DECODE_BASE_H
#define LZVN_DECODE_BASE_H
#include "lzfse_internal.h"
/*! @abstract Base decoder state. */
typedef struct {
// Decoder I/O
// Next byte to read in source buffer
const unsigned char *src;
// Next byte after source buffer
const unsigned char *src_end;
// Next byte to write in destination buffer (by decoder)
unsigned char *dst;
// Valid range for destination buffer is [dst_begin, dst_end - 1]
unsigned char *dst_begin;
unsigned char *dst_end;
// Next byte to read in destination buffer (modified by caller)
unsigned char *dst_current;
// Decoder state
// Partially expanded match, or 0,0,0.
// In that case, src points to the next literal to copy, or the next op-code
// if L==0.
size_t L, M, D;
// Distance for last emitted match, or 0
lzvn_offset d_prev;
// Did we decode end-of-stream?
int end_of_stream;
} lzvn_decoder_state;
/*! @abstract Decode source to destination.
* Updates \p state (src,dst,d_prev). */
void lzvn_decode(lzvn_decoder_state *state);
#endif // LZVN_DECODE_BASE_H
+593
查看文件
@@ -0,0 +1,593 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// LZVN low-level encoder
#include "lzvn_encode_base.h"
#if defined(_MSC_VER) && !defined(__clang__)
# define restrict __restrict
#endif
// ===============================================================
// Coarse/fine copy, non overlapping buffers
/*! @abstract Copy at least \p nbytes bytes from \p src to \p dst, by blocks
* of 8 bytes (may go beyond range). No overlap.
* @return \p dst + \p nbytes. */
static inline unsigned char *lzvn_copy64(unsigned char *restrict dst,
const unsigned char *restrict src,
size_t nbytes) {
for (size_t i = 0; i < nbytes; i += 8)
store8(dst + i, load8(src + i));
return dst + nbytes;
}
/*! @abstract Copy exactly \p nbytes bytes from \p src to \p dst (respects range).
* No overlap.
* @return \p dst + \p nbytes. */
static inline unsigned char *lzvn_copy8(unsigned char *restrict dst,
const unsigned char *restrict src,
size_t nbytes) {
for (size_t i = 0; i < nbytes; i++)
dst[i] = src[i];
return dst + nbytes;
}
/*! @abstract Emit (L,0,0) instructions (final literal).
* We read at most \p L bytes from \p p.
* @param p input stream
* @param q1 the first byte after the output buffer.
* @return pointer to the next output, <= \p q1.
* @return \p q1 if output is full. In that case, output will be partially invalid.
*/
static inline unsigned char *emit_literal(const unsigned char *p,
unsigned char *q, unsigned char *q1,
size_t L) {
size_t x;
while (L > 15) {
x = L < 271 ? L : 271;
if (q + x + 10 >= q1)
goto OUT_FULL;
store2(q, 0xE0 + ((x - 16) << 8));
q += 2;
L -= x;
q = lzvn_copy8(q, p, x);
p += x;
}
if (L > 0) {
if (q + L + 10 >= q1)
goto OUT_FULL;
*q++ = 0xE0 + L; // 1110LLLL
q = lzvn_copy8(q, p, L);
}
return q;
OUT_FULL:
return q1;
}
/*! @abstract Emit (L,M,D) instructions. M>=3.
* @param p input stream pointing to the beginning of the literal. We read at
* most \p L+4 bytes from \p p.
* @param q1 the first byte after the output buffer.
* @return pointer to the next output, <= \p q1.
* @return \p q1 if output is full. In that case, output will be partially invalid.
*/
static inline unsigned char *emit(const unsigned char *p, unsigned char *q,
unsigned char *q1, size_t L, size_t M,
size_t D, size_t D_prev) {
size_t x;
while (L > 15) {
x = L < 271 ? L : 271;
if (q + x + 10 >= q1)
goto OUT_FULL;
store2(q, 0xE0 + ((x - 16) << 8));
q += 2;
L -= x;
q = lzvn_copy64(q, p, x);
p += x;
}
if (L > 3) {
if (q + L + 10 >= q1)
goto OUT_FULL;
*q++ = 0xE0 + L; // 1110LLLL
q = lzvn_copy64(q, p, L);
p += L;
L = 0;
}
x = M <= 10 - 2 * L ? M : 10 - 2 * L; // x = min(10-2*L,M)
M -= x;
x -= 3; // M = (x+3) + M' max value for x is 7-2*L
// Here L<4 literals remaining, we read them here
uint32_t literal = load4(p);
// P is not accessed after this point
// Relaxed capacity test covering all cases
if (q + 8 >= q1)
goto OUT_FULL;
if (D == D_prev) {
if (L == 0) {
*q++ = 0xF0 + (x + 3); // XM!
} else {
*q++ = (L << 6) + (x << 3) + 6; // LLxxx110
}
store4(q, literal);
q += L;
} else if (D < 2048 - 2 * 256) {
// Short dist D>>8 in 0..5
*q++ = (D >> 8) + (L << 6) + (x << 3); // LLxxxDDD
*q++ = D & 0xFF;
store4(q, literal);
q += L;
} else if (D >= (1 << 14) || M == 0 || (x + 3) + M > 34) {
// Long dist
*q++ = (L << 6) + (x << 3) + 7;
store2(q, D);
q += 2;
store4(q, literal);
q += L;
} else {
// Medium distance
x += M;
M = 0;
*q++ = 0xA0 + (x >> 2) + (L << 3);
store2(q, D << 2 | (x & 3));
q += 2;
store4(q, literal);
q += L;
}
// Issue remaining match
while (M > 15) {
if (q + 2 >= q1)
goto OUT_FULL;
x = M < 271 ? M : 271;
store2(q, 0xf0 + ((x - 16) << 8));
q += 2;
M -= x;
}
if (M > 0) {
if (q + 1 >= q1)
goto OUT_FULL;
*q++ = 0xF0 + M; // M = 0..15
}
return q;
OUT_FULL:
return q1;
}
// ===============================================================
// Conversions
/*! @abstract Return 32-bit value to store for offset x. */
static inline int32_t offset_to_s32(lzvn_offset x) { return (int32_t)x; }
/*! @abstract Get offset from 32-bit stored value x. */
static inline lzvn_offset offset_from_s32(int32_t x) { return (lzvn_offset)x; }
// ===============================================================
// Hash and Matching
/*! @abstract Get hash in range \c [0,LZVN_ENCODE_HASH_VALUES-1] from 3 bytes in i. */
static inline uint32_t hash3i(uint32_t i) {
i &= 0xffffff; // truncate to 24-bit input (slightly increases compression ratio)
uint32_t h = (i * (1 + (1 << 6) + (1 << 12))) >> 12;
return h & (LZVN_ENCODE_HASH_VALUES - 1);
}
/*! @abstract Return the number [0, 4] of zero bytes in \p x, starting from the
* least significant byte. */
static inline lzvn_offset trailing_zero_bytes(uint32_t x) {
return (x == 0) ? 4 : (__builtin_ctzl(x) >> 3);
}
/*! @abstract Return the number [0, 4] of matching chars between values at
* \p src+i and \p src+j, starting from the least significant byte.
* Assumes we can read 4 chars from each position. */
static inline lzvn_offset nmatch4(const unsigned char *src, lzvn_offset i,
lzvn_offset j) {
uint32_t vi = load4(src + i);
uint32_t vj = load4(src + j);
return trailing_zero_bytes(vi ^ vj);
}
/*! @abstract Check if l_begin, m_begin, m0_begin (m0_begin < m_begin) can be
* expanded to a match of length at least 3.
* @param m_begin new string to match.
* @param m0_begin candidate old string.
* @param src source buffer, with valid indices src_begin <= i < src_end.
* (src_begin may be <0)
* @return If a match can be found, return 1 and set all \p match fields,
* otherwise return 0.
* @note \p *match should be 0 before the call. */
static inline int lzvn_find_match(const unsigned char *src,
lzvn_offset src_begin,
lzvn_offset src_end, lzvn_offset l_begin,
lzvn_offset m0_begin, lzvn_offset m_begin,
lzvn_match_info *match) {
lzvn_offset n = nmatch4(src, m_begin, m0_begin);
if (n < 3)
return 0; // no match
lzvn_offset D = m_begin - m0_begin; // actual distance
if (D <= 0 || D > LZVN_ENCODE_MAX_DISTANCE)
return 0; // distance out of range
// Expand forward
lzvn_offset m_end = m_begin + n;
while (n == 4 && m_end + 4 < src_end) {
n = nmatch4(src, m_end, m_end - D);
m_end += n;
}
// Expand backwards over literal
while (m0_begin > src_begin && m_begin > l_begin &&
src[m_begin - 1] == src[m0_begin - 1]) {
m0_begin--;
m_begin--;
}
// OK, we keep it, update MATCH
lzvn_offset M = m_end - m_begin; // match length
match->m_begin = m_begin;
match->m_end = m_end;
match->K = M - ((D < 0x600) ? 2 : 3);
match->M = M;
match->D = D;
return 1; // OK
}
/*! @abstract Same as lzvn_find_match, but we already know that N bytes do
* match (N<=4). */
static inline int lzvn_find_matchN(const unsigned char *src,
lzvn_offset src_begin,
lzvn_offset src_end, lzvn_offset l_begin,
lzvn_offset m0_begin, lzvn_offset m_begin,
lzvn_offset n, lzvn_match_info *match) {
// We can skip the first comparison on 4 bytes
if (n < 3)
return 0; // no match
lzvn_offset D = m_begin - m0_begin; // actual distance
if (D <= 0 || D > LZVN_ENCODE_MAX_DISTANCE)
return 0; // distance out of range
// Expand forward
lzvn_offset m_end = m_begin + n;
while (n == 4 && m_end + 4 < src_end) {
n = nmatch4(src, m_end, m_end - D);
m_end += n;
}
// Expand backwards over literal
while (m0_begin > src_begin && m_begin > l_begin &&
src[m_begin - 1] == src[m0_begin - 1]) {
m0_begin--;
m_begin--;
}
// OK, we keep it, update MATCH
lzvn_offset M = m_end - m_begin; // match length
match->m_begin = m_begin;
match->m_end = m_end;
match->K = M - ((D < 0x600) ? 2 : 3);
match->M = M;
match->D = D;
return 1; // OK
}
// ===============================================================
// Encoder Backend
/*! @abstract Emit a match and update state.
* @return number of bytes written to \p dst. May be 0 if there is no more space
* in \p dst to emit the match. */
static inline lzvn_offset lzvn_emit_match(lzvn_encoder_state *state,
lzvn_match_info match) {
size_t L = (size_t)(match.m_begin - state->src_literal); // literal count
size_t M = (size_t)match.M; // match length
size_t D = (size_t)match.D; // match distance
size_t D_prev = (size_t)state->d_prev; // previously emitted match distance
unsigned char *dst = emit(state->src + state->src_literal, state->dst,
state->dst_end, L, M, D, D_prev);
// Check if DST is full
if (dst >= state->dst_end) {
return 0; // FULL
}
// Update state
lzvn_offset dst_used = dst - state->dst;
state->d_prev = match.D;
state->dst = dst;
state->src_literal = match.m_end;
return dst_used;
}
/*! @abstract Emit a n-bytes literal and update state.
* @return number of bytes written to \p dst. May be 0 if there is no more space
* in \p dst to emit the literal. */
static inline lzvn_offset lzvn_emit_literal(lzvn_encoder_state *state,
lzvn_offset n) {
size_t L = (size_t)n;
unsigned char *dst = emit_literal(state->src + state->src_literal, state->dst,
state->dst_end, L);
// Check if DST is full
if (dst >= state->dst_end)
return 0; // FULL
// Update state
lzvn_offset dst_used = dst - state->dst;
state->dst = dst;
state->src_literal += n;
return dst_used;
}
/*! @abstract Emit end-of-stream and update state.
* @return number of bytes written to \p dst. May be 0 if there is no more space
* in \p dst to emit the instruction. */
static inline lzvn_offset lzvn_emit_end_of_stream(lzvn_encoder_state *state) {
// Do we have 8 byte in dst?
if (state->dst_end < state->dst + 8)
return 0; // FULL
// Insert end marker and update state
store8(state->dst, 0x06); // end-of-stream command
state->dst += 8;
return 8; // dst_used
}
// ===============================================================
// Encoder Functions
/*! @abstract Initialize encoder table in \p state, uses current I/O parameters. */
static inline void lzvn_init_table(lzvn_encoder_state *state) {
lzvn_offset index = -LZVN_ENCODE_MAX_DISTANCE; // max match distance
if (index < state->src_begin)
index = state->src_begin;
uint32_t value = load4(state->src + index);
lzvn_encode_entry_type e;
for (int i = 0; i < 4; i++) {
e.indices[i] = offset_to_s32(index);
e.values[i] = value;
}
for (int u = 0; u < LZVN_ENCODE_HASH_VALUES; u++)
state->table[u] = e; // fill entire table
}
void lzvn_encode(lzvn_encoder_state *state) {
const lzvn_match_info NO_MATCH = {0};
for (; state->src_current < state->src_current_end; state->src_current++) {
// Get 4 bytes at src_current
uint32_t vi = load4(state->src + state->src_current);
// Compute new hash H at position I, and push value into position table
int h = hash3i(vi); // index of first entry
// Read table entries for H
lzvn_encode_entry_type e = state->table[h];
// Update entry with index=current and value=vi
lzvn_encode_entry_type updated_e; // rotate values, so we will replace the oldest
updated_e.indices[0] = offset_to_s32(state->src_current);
updated_e.indices[1] = e.indices[0];
updated_e.indices[2] = e.indices[1];
updated_e.indices[3] = e.indices[2];
updated_e.values[0] = vi;
updated_e.values[1] = e.values[0];
updated_e.values[2] = e.values[1];
updated_e.values[3] = e.values[2];
// Do not check matches if still in previously emitted match
if (state->src_current < state->src_literal)
goto after_emit;
// Update best with candidate if better
#define UPDATE(best, candidate) \
do { \
if (candidate.K > best.K || \
((candidate.K == best.K) && (candidate.m_end > best.m_end + 1))) { \
best = candidate; \
} \
} while (0)
// Check candidate. Keep if better.
#define CHECK_CANDIDATE(ik, nk) \
do { \
lzvn_match_info m1; \
if (lzvn_find_matchN(state->src, state->src_begin, state->src_end, \
state->src_literal, ik, state->src_current, nk, &m1)) { \
UPDATE(incoming, m1); \
} \
} while (0)
// Emit match M. Return if we don't have enough space in the destination buffer
#define EMIT_MATCH(m) \
do { \
if (lzvn_emit_match(state, m) == 0) \
return; \
} while (0)
// Emit literal of length L. Return if we don't have enough space in the
// destination buffer
#define EMIT_LITERAL(l) \
do { \
if (lzvn_emit_literal(state, l) == 0) \
return; \
} while (0)
lzvn_match_info incoming = NO_MATCH;
// Check candidates in order (closest first)
uint32_t diffs[4];
for (int k = 0; k < 4; k++)
diffs[k] = e.values[k] ^ vi; // XOR, 0 if equal
lzvn_offset ik; // index
lzvn_offset nk; // match byte count
// The values stored in e.xyzw are 32-bit signed indices, extended to signed
// type lzvn_offset
ik = offset_from_s32(e.indices[0]);
nk = trailing_zero_bytes(diffs[0]);
CHECK_CANDIDATE(ik, nk);
ik = offset_from_s32(e.indices[1]);
nk = trailing_zero_bytes(diffs[1]);
CHECK_CANDIDATE(ik, nk);
ik = offset_from_s32(e.indices[2]);
nk = trailing_zero_bytes(diffs[2]);
CHECK_CANDIDATE(ik, nk);
ik = offset_from_s32(e.indices[3]);
nk = trailing_zero_bytes(diffs[3]);
CHECK_CANDIDATE(ik, nk);
// Check candidate at previous distance
if (state->d_prev != 0) {
lzvn_match_info m1;
if (lzvn_find_match(state->src, state->src_begin, state->src_end,
state->src_literal, state->src_current - state->d_prev,
state->src_current, &m1)) {
m1.K = m1.M - 1; // fix K for D_prev
UPDATE(incoming, m1);
}
}
// Here we have the best candidate in incoming, may be NO_MATCH
// If no incoming match, and literal backlog becomes too high, emit pending
// match, or literals if there is no pending match
if (incoming.M == 0) {
if (state->src_current - state->src_literal >=
LZVN_ENCODE_MAX_LITERAL_BACKLOG) // at this point, we always have
// current >= literal
{
if (state->pending.M != 0) {
EMIT_MATCH(state->pending);
state->pending = NO_MATCH;
} else {
EMIT_LITERAL(271); // emit long literal (271 is the longest literal size we allow)
}
}
goto after_emit;
}
if (state->pending.M == 0) {
// NOTE. Here, we can also emit incoming right away. It will make the
// encoder 1.5x faster, at a cost of ~10% lower compression ratio:
// EMIT_MATCH(incoming);
// state->pending = NO_MATCH;
// No pending match, emit nothing, keep incoming
state->pending = incoming;
} else {
// Here we have both incoming and pending
if (state->pending.m_end <= incoming.m_begin) {
// No overlap: emit pending, keep incoming
EMIT_MATCH(state->pending);
state->pending = incoming;
} else {
// If pending is better, emit pending and discard incoming.
// Otherwise, emit incoming and discard pending.
if (incoming.K > state->pending.K)
state->pending = incoming;
EMIT_MATCH(state->pending);
state->pending = NO_MATCH;
}
}
after_emit:
// We commit state changes only after we tried to emit instructions, so we
// can restart in the same state in case dst was full and we quit the loop.
state->table[h] = updated_e;
} // i loop
// Do not emit pending match here. We do it only at the end of stream.
}
// ===============================================================
// API entry points
size_t lzvn_encode_scratch_size(void) { return LZVN_ENCODE_WORK_SIZE; }
static size_t lzvn_encode_partial(void *__restrict dst, size_t dst_size,
const void *__restrict src, size_t src_size,
size_t *src_used, void *__restrict work) {
// Min size checks to avoid accessing memory outside buffers.
if (dst_size < LZVN_ENCODE_MIN_DST_SIZE) {
*src_used = 0;
return 0;
}
// Max input size check (limit to offsets on uint32_t).
if (src_size > LZVN_ENCODE_MAX_SRC_SIZE) {
src_size = LZVN_ENCODE_MAX_SRC_SIZE;
}
// Setup encoder state
lzvn_encoder_state state;
memset(&state, 0, sizeof(state));
state.src = src;
state.src_begin = 0;
state.src_end = (lzvn_offset)src_size;
state.src_literal = 0;
state.src_current = 0;
state.dst = dst;
state.dst_begin = dst;
state.dst_end = (unsigned char *)dst + dst_size - 8; // reserve 8 bytes for end-of-stream
state.table = work;
// Do not encode if the input buffer is too small. We'll emit a literal instead.
if (src_size >= LZVN_ENCODE_MIN_SRC_SIZE) {
state.src_current_end = (lzvn_offset)src_size - LZVN_ENCODE_MIN_MARGIN;
lzvn_init_table(&state);
lzvn_encode(&state);
}
// No need to test the return value: src_literal will not be updated on failure,
// and we will fail later.
lzvn_emit_literal(&state, state.src_end - state.src_literal);
// Restore original size, so end-of-stream always succeeds, and emit it
state.dst_end = (unsigned char *)dst + dst_size;
lzvn_emit_end_of_stream(&state);
*src_used = state.src_literal;
return (size_t)(state.dst - state.dst_begin);
}
size_t lzvn_encode_buffer(void *__restrict dst, size_t dst_size,
const void *__restrict src, size_t src_size,
void *__restrict work) {
size_t src_used = 0;
size_t dst_used =
lzvn_encode_partial(dst, dst_size, src, src_size, &src_used, work);
if (src_used != src_size)
return 0; // could not encode entire input stream = fail
return dst_used; // return encoded size
}
+116
查看文件
@@ -0,0 +1,116 @@
/*
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// LZVN low-level encoder
#ifndef LZVN_ENCODE_BASE_H
#define LZVN_ENCODE_BASE_H
#include "lzfse_internal.h"
// ===============================================================
// Types and Constants
#define LZVN_ENCODE_HASH_BITS \
14 // number of bits returned by the hash function [10, 16]
#define LZVN_ENCODE_OFFSETS_PER_HASH \
4 // stored offsets stack for each hash value, MUST be 4
#define LZVN_ENCODE_HASH_VALUES \
(1 << LZVN_ENCODE_HASH_BITS) // number of entries in hash table
#define LZVN_ENCODE_MAX_DISTANCE \
0xffff // max match distance we can represent with LZVN encoding, MUST be
// 0xFFFF
#define LZVN_ENCODE_MIN_MARGIN \
8 // min number of bytes required between current and end during encoding,
// MUST be >= 8
#define LZVN_ENCODE_MAX_LITERAL_BACKLOG \
400 // if the number of pending literals exceeds this size, emit a long
// literal, MUST be >= 271
/*! @abstract Type of table entry. */
typedef struct {
int32_t indices[4]; // signed indices in source buffer
uint32_t values[4]; // corresponding 32-bit values
} lzvn_encode_entry_type;
// Work size
#define LZVN_ENCODE_WORK_SIZE \
(LZVN_ENCODE_HASH_VALUES * sizeof(lzvn_encode_entry_type))
/*! @abstract Match */
typedef struct {
lzvn_offset m_begin; // beginning of match, current position
lzvn_offset m_end; // end of match, this is where the next literal would begin
// if we emit the entire match
lzvn_offset M; // match length M: m_end - m_begin
lzvn_offset D; // match distance D
lzvn_offset K; // match gain: M - distance storage (L not included)
} lzvn_match_info;
// ===============================================================
// Internal encoder state
/*! @abstract Base encoder state and I/O. */
typedef struct {
// Encoder I/O
// Source buffer
const unsigned char *src;
// Valid range in source buffer: we can access src[i] for src_begin <= i <
// src_end. src_begin may be negative.
lzvn_offset src_begin;
lzvn_offset src_end;
// Next byte to process in source buffer
lzvn_offset src_current;
// Next byte after the last byte to process in source buffer. We MUST have:
// src_current_end + 8 <= src_end.
lzvn_offset src_current_end;
// Next byte to encode in source buffer, may be before or after src_current.
lzvn_offset src_literal;
// Next byte to write in destination buffer
unsigned char *dst;
// Valid range in destination buffer: [dst_begin, dst_end - 1]
unsigned char *dst_begin;
unsigned char *dst_end;
// Encoder state
// Pending match
lzvn_match_info pending;
// Distance for last emitted match, or 0
lzvn_offset d_prev;
// Hash table used to find matches. Stores LZVN_ENCODE_OFFSETS_PER_HASH 32-bit
// signed indices in the source buffer, and the corresponding 4-byte values.
// The number of entries in the table is LZVN_ENCODE_HASH_VALUES.
lzvn_encode_entry_type *table;
} lzvn_encoder_state;
/*! @abstract Encode source to destination.
* Update \p state.
* The call ensures \c src_literal is never left too far behind \c src_current. */
void lzvn_encode(lzvn_encoder_state *state);
#endif // LZVN_ENCODE_BASE_H
+53
查看文件
@@ -0,0 +1,53 @@
get_filename_component(INPUT_NAME "${INPUT}" NAME)
set(LZFSE_TMP_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-tmp")
set(COMPRESSED_FILE "${LZFSE_TMP_DIR}/${INPUT_NAME}.lzfse")
set(DECOMPRESSED_FILE "${LZFSE_TMP_DIR}/${INPUT_NAME}")
file(MAKE_DIRECTORY "${LZFSE_TMP_DIR}")
if(EXISTS COMPRESSED_FILE)
message(FATAL_ERROR "${COMPRESSED_FILE} exists")
file(REMOVE ${COMPRESSED_FILE})
endif()
if(EXISTS DECOMPRESSED_FILE)
message(FATAL_ERROR "${DECOMPRESSED_FILE} exists")
file(REMOVE ${DECOMPRESSED_FILE})
endif()
execute_process(
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/.."
COMMAND ${LZFSE_WRAPPER} ${LZFSE_CLI} -encode -i ${INPUT} -o ${COMPRESSED_FILE}
RESULT_VARIABLE result
ERROR_VARIABLE result_stderr)
if(result)
message(FATAL_ERROR "Compression failed: ${result_stderr}")
endif()
execute_process(
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/.."
COMMAND ${LZFSE_WRAPPER} ${LZFSE_CLI} -decode -i ${COMPRESSED_FILE} -o ${DECOMPRESSED_FILE}
RESULT_VARIABLE result)
if(result)
message(FATAL_ERROR "Decompression failed")
endif()
function(test_file_equality f1 f2)
if(NOT CMAKE_VERSION VERSION_LESS 2.8.7)
file(SHA512 "${f1}" f1_cs)
file(SHA512 "${f2}" f2_cs)
if(NOT "${f1_cs}" STREQUAL "${f2_cs}")
message(FATAL_ERROR "Files do not match")
endif()
else()
file(READ "${f1}" f1_contents)
file(READ "${f2}" f2_contents)
if(NOT "${f1_contents}" STREQUAL "${f2_contents}")
message(FATAL_ERROR "Files do not match")
endif()
endif()
endfunction()
test_file_equality("${INPUT}" "${DECOMPRESSED_FILE}")
file(REMOVE ${COMPRESSED_FILE})
file(REMOVE ${DECOMPRESSED_FILE})
+483
查看文件
@@ -0,0 +1,483 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <memory>
#include <fstream>
#include <iostream>
#include <iomanip>
#include <cstring>
#include <ApfsLib/Device.h>
#include <ApfsLib/Util.h>
#include <ApfsLib/DiskStruct.h>
#include <ApfsLib/BlockDumper.h>
#include <ApfsLib/GptPartitionMap.h>
#include "Dumper.h"
#ifdef __linux__
#include <signal.h>
#endif
#undef RAW_VERBOSE
constexpr size_t BLOCKSIZE = 0x1000;
volatile bool g_abort = 0;
void DumpBlockTrunc(std::ostream &os, const uint8_t *data)
{
unsigned int sz = BLOCKSIZE - 1;
while (sz > 0 && data[sz] == 0)
sz--;
sz = (sz + 0x10) & 0xFFFFFFF0;
DumpHex(os, data, sz);
}
#if 0
void MapBlocks(std::ostream &os, Device &dev, uint64_t bid_start, uint64_t bcnt)
{
using namespace std;
uint64_t bid;
uint8_t block[BLOCKSIZE];
const APFS_ObjHeader * const blk = reinterpret_cast<const APFS_ObjHeader *>(block);
const APFS_TableHeader * const tbl = reinterpret_cast<const APFS_TableHeader *>(block + sizeof(APFS_ObjHeader));
bool last_was_used = false;
os << hex << uppercase << setfill('0');
os << "[Block] | oid | xid | type | subtype | Page | Levl | Entries | Description" << endl;
os << "---------+----------+----------+----------+----------+------+------+----------+---------------------------------" << endl;
for (bid = 0; bid < bcnt && !g_abort; bid++)
{
if ((bid & 0xFFF) == 0)
{
std::cout << '.';
std::cout.flush();
}
dev.Read(block, (bid_start + bid) * BLOCKSIZE, BLOCKSIZE);
if (IsEmptyBlock(block, BLOCKSIZE))
{
if (last_was_used)
os << "---------+----------+----------+----------+----------+------+------+----------+ Empty" << endl;
last_was_used = false;
continue;
}
if (VerifyBlock(block, BLOCKSIZE))
{
os << setw(8) << bid << " | ";
os << setw(8) << blk->oid << " | ";
os << setw(8) << blk->xid << " | ";
os << setw(8) << blk->type << " | ";
os << setw(8) << blk->subtype << " | ";
os << setw(4) << tbl->page << " | ";
os << setw(4) << tbl->level << " | ";
os << setw(8) << tbl->entries_cnt << " | ";
os << BlockDumper::GetNodeType(blk->type, blk->subtype);
if (APFS_OBJ_TYPE(blk->type) == 2)
os << " [Root]";
os << endl;
last_was_used = true;
}
else
{
os << setw(8) << bid;
os << " | | | | | | | | Data" << endl;
last_was_used = true;
}
}
os << endl;
}
void ScanBlocks(std::ostream &os, Device &dev, uint64_t bid_start, uint64_t bcnt)
{
BlockDumper bd(os, BLOCKSIZE);
uint64_t bid;
uint8_t block[BLOCKSIZE];
bd.SetTextFlags(0x00);
for (bid = 0; bid < bcnt && !g_abort; bid++)
{
if ((bid & 0xFFF) == 0)
{
std::cout << '.';
std::cout.flush();
}
dev.Read(block, (bid_start + bid) * BLOCKSIZE, BLOCKSIZE);
if (IsEmptyBlock(block, BLOCKSIZE))
continue;
if (VerifyBlock(block, BLOCKSIZE))
bd.DumpNode(block, bid);
else
{
#if 0
os << std::hex << std::setw(16) << blk_nr << std::endl;
DumpBlockTrunc(os, block);
os << std::endl;
os << "===========================================================================================================================" << std::endl;
os << std::endl;
#endif
}
}
}
void DumpSpaceman(std::ostream &os, Device &dev, uint64_t bid_start, uint64_t bcnt)
{
uint8_t sb[BLOCKSIZE];
uint8_t data[BLOCKSIZE];
BlockDumper bd(os, BLOCKSIZE);
const APFS_ObjHeader *bhdr;
const APFS_Spaceman *sm;
const APFS_NX_Superblock *nxsb;
uint64_t bid;
uint64_t cnt;
uint64_t smbmp_bid = 0;
uint64_t smbmp_cnt = 0;
uint64_t volbmp_bid = 0;
uint64_t volbmp_cnt = 0;
(void)bcnt;
if (!dev.Read(sb, bid_start * BLOCKSIZE, BLOCKSIZE))
return;
if (!VerifyBlock(sb, BLOCKSIZE))
return;
nxsb = reinterpret_cast<const APFS_NX_Superblock *>(sb);
if (nxsb->hdr.type != 0x80000001)
return;
if (nxsb->nx_magic != 0x4253584E)
return;
bd.DumpNode(sb, 0);
os << std::endl;
os << "Dumping Superblock Area" << std::endl;
os << std::endl;
bid = nxsb->nx_xp_desc_base;
cnt = nxsb->nx_xp_desc_blocks;
while (cnt > 0)
{
dev.Read(data, (bid_start + bid) * BLOCKSIZE, BLOCKSIZE);
bd.DumpNode(data, bid);
cnt--;
bid++;
}
os << std::endl;
os << "Dumping Spaceman Area" << std::endl;
os << std::endl;
bid = nxsb->nx_xp_data_base;
cnt = nxsb->nx_xp_data_blocks;
while (cnt > 0)
{
dev.Read(data, (bid_start + bid) * BLOCKSIZE, BLOCKSIZE);
bd.DumpNode(data, bid);
cnt--;
bid++;
bhdr = reinterpret_cast<const APFS_ObjHeader *>(data);
if (bhdr->type == 0x80000005)
{
sm = reinterpret_cast<const APFS_Spaceman *>(data);
smbmp_bid = sm->ip_bm_base_address;
smbmp_cnt = sm->ip_bitmap_block_count;
volbmp_bid = sm->ip_base_address;
volbmp_cnt = sm->ip_block_count;
}
}
bid = smbmp_bid;
cnt = smbmp_cnt;
while (cnt > 0)
{
dev.Read(data, (bid_start + bid) * BLOCKSIZE, BLOCKSIZE);
bd.DumpNode(data, bid);
cnt--;
bid++;
}
bid = volbmp_bid;
cnt = volbmp_cnt;
while (cnt > 0)
{
dev.Read(data, (bid_start + bid) * BLOCKSIZE, BLOCKSIZE);
bd.DumpNode(data, bid);
cnt--;
bid++;
}
}
#endif
static void ctrl_c_handler(int sig)
{
(void)sig;
g_abort = true;
}
#if 0
int main(int argc, const char *argv[])
{
if (argc < 3)
{
std::cerr << "Syntax: apfs-dump file.img output.txt [map.txt]" << std::endl;
return 1;
}
Device *dev;
std::ofstream os;
uint64_t bid_start = 0;
uint64_t bcnt = 0;
g_debug = 255;
#if defined(__linux__) || defined(__APPLE__)
signal(SIGINT, ctrl_c_handler);
#endif
dev = Device::OpenDevice(argv[1]);
if (!dev)
{
std::cerr << "Device " << argv[1] << " not found." << std::endl;
return 2;
}
{
GptPartitionMap pmap;
if (pmap.LoadAndVerify(*dev))
{
int partid = pmap.FindFirstAPFSPartition();
if (partid >= 0)
{
std::cout << "Dumping EFI partition" << std::endl;
pmap.GetPartitionOffsetAndSize(partid, bid_start, bcnt);
bid_start /= BLOCKSIZE;
bcnt /= BLOCKSIZE;
}
}
}
if (bcnt == 0)
bcnt = dev->GetSize() / BLOCKSIZE;
if (argc > 3)
{
os.open(argv[3]);
if (!os.is_open())
{
std::cerr << "Could not open output file " << argv[3] << std::endl;
dev->Close();
delete dev;
return 3;
}
MapBlocks(os, *dev, bid_start, bcnt);
os.close();
}
os.open(argv[2]);
if (!os.is_open())
{
std::cerr << "Could not open output file " << argv[2] << std::endl;
dev->Close();
delete dev;
return 3;
}
DumpSpaceman(os, *dev, bid_start, bcnt);
ScanBlocks(os, *dev, bid_start, bcnt);
dev->Close();
os.close();
delete dev;
return 0;
}
#else
void usage()
{
std::cerr << "Syntax:" << std::endl;
std::cerr << "apfs-dump [-map mapfile.txt] file.img output.txt" << std::endl;
std::cerr << "apfs-dump [-map mapfile.txt] -fusion main.img tier2.img output.txt" << std::endl;
}
int main(int argc, const char *argv[])
{
const char *name_dev_main = 0;
const char *name_dev_tier2 = 0;
const char *name_map = 0;
const char *name_output = 0;
bool use_fusion = false;
int n;
int idx = 0;
for (n = 1; n < argc; n++)
{
if (!strcmp(argv[n], "-map"))
{
n++;
if (n >= argc)
{
usage();
return 1;
}
name_map = argv[n];
}
else if (!strcmp(argv[n], "-fusion"))
use_fusion = true;
else
{
switch (idx)
{
case 0:
name_dev_main = argv[n];
printf("main: %s\n", name_dev_main);
if (use_fusion)
idx++;
else
idx += 2;
break;
case 1:
name_dev_tier2 = argv[n];
printf("tier2: %s\n", name_dev_tier2);
idx++;
break;
case 2:
name_output = argv[n];
printf("out: %s\n", name_output);
idx++;
break;
default:
usage();
return 1;
break;
}
}
}
if ((name_output == 0) || (use_fusion && name_dev_tier2 == 0))
{
usage();
return 1;
}
std::unique_ptr<Device> dev_main;
std::unique_ptr<Device> dev_tier2;
std::ofstream os;
g_debug = 255;
#if defined(__linux__) || defined(__APPLE__)
signal(SIGINT, ctrl_c_handler);
#endif
dev_main.reset(Device::OpenDevice(name_dev_main));
if (use_fusion)
dev_tier2.reset(Device::OpenDevice(name_dev_tier2));
if (!dev_main)
{
std::cerr << "Device " << name_dev_main << " not found." << std::endl;
return 2;
}
if (use_fusion && !dev_tier2)
{
std::cerr << "Device " << name_dev_tier2 << " not found." << std::endl;
return 2;
}
{
Dumper dmp(dev_main.get(), dev_tier2.get());
if (!dmp.Initialize())
return -1;
#if 1
if (name_map)
{
os.open(name_map);
if (!os.is_open())
{
std::cerr << "Could not open map file " << name_map << std::endl;
dev_main->Close();
if (dev_tier2)
dev_tier2->Close();
return 3;
}
dmp.DumpBlockList(os);
os.close();
}
#endif
os.open(name_output);
if (!os.is_open())
{
std::cerr << "Could not open output file " << name_output << std::endl;
dev_main->Close();
if (dev_tier2)
dev_tier2->Close();
return 3;
}
dmp.DumpContainer(os);
}
dev_main->Close();
if (dev_tier2)
dev_tier2->Close();
os.close();
return 0;
}
#endif
+476
查看文件
@@ -0,0 +1,476 @@
#include <iostream>
#include <iomanip>
#include <ApfsLib/BlockDumper.h>
#include <ApfsLib/Device.h>
#include <ApfsLib/DiskStruct.h>
#include <ApfsLib/GptPartitionMap.h>
#include <ApfsLib/Util.h>
#include "Dumper.h"
Dumper::Dumper(Device *dev_main, Device *dev_tier2)
{
m_dev_main = dev_main;
m_dev_tier2 = dev_tier2;
m_base_main = 0;
m_size_main = 0;
m_base_tier2 = 0;
m_size_tier2 = 0;
m_blocksize = 0;
m_is_encrypted = false;
}
Dumper::~Dumper()
{
}
bool Dumper::Initialize()
{
m_base_main = 0;
m_size_main = 0;
m_base_tier2 = 0;
m_size_tier2 = 0;
m_blocksize = 0;
GptPartitionMap pmap;
if (pmap.LoadAndVerify(*m_dev_main))
{
int partid = pmap.FindFirstAPFSPartition();
if (partid >= 0)
{
std::cout << "Dumping EFI partition on main" << std::endl;
pmap.GetPartitionOffsetAndSize(partid, m_base_main, m_size_main);
}
}
if (m_size_main == 0)
m_size_main = m_dev_main->GetSize();
if (m_size_main == 0)
return false;
if (m_dev_tier2)
{
if (pmap.LoadAndVerify(*m_dev_tier2))
{
int partid = pmap.FindFirstAPFSPartition();
if (partid >= 0)
{
std::cout << "Dumping EFI partition on tier2" << std::endl;
pmap.GetPartitionOffsetAndSize(partid, m_base_tier2, m_size_tier2);
}
}
if (m_size_tier2 == 0)
m_size_tier2 = m_dev_tier2->GetSize();
if (m_size_tier2 == 0)
return false;
}
std::vector<uint8_t> nx_data;
const nx_superblock_t *nx;
nx_data.resize(0x1000);
m_dev_main->Read(nx_data.data(), m_base_main, 0x1000);
nx = reinterpret_cast<const nx_superblock_t *>(nx_data.data());
if (nx->nx_magic != NX_MAGIC)
{
std::cerr << "Could not load NX superblock. Invalid magic number." << std::endl;
return false;
}
m_blocksize = nx->nx_block_size;
if (m_blocksize != 0x1000)
{
nx_data.resize(m_blocksize);
nx = reinterpret_cast<const nx_superblock_t *>(nx_data.data());
m_dev_main->Read(nx_data.data(), m_base_main, m_blocksize);
}
if (!VerifyBlock(nx_data.data(), m_blocksize))
{
std::cerr << "NX superblock checksum incorrect!" << std::endl;
return false;
}
return true;
}
bool Dumper::DumpContainer(std::ostream &os)
{
BlockDumper bd(os, m_blocksize);
std::vector<uint8_t> nx_data;
std::vector<uint8_t> cpm_data;
std::vector<uint8_t> sm_data;
std::vector<uint8_t> cxb_data;
std::vector<uint8_t> bmp_data;
std::vector<uint8_t> blk_data;
const nx_superblock_t *nx = nullptr;
const checkpoint_map_phys_t *cpm = nullptr;
const spaceman_phys_t *sm = nullptr;
const chunk_info_block_t *cib = nullptr;
const cib_addr_block_t *cab = nullptr;
const checkpoint_mapping_t *cm = nullptr;
uint64_t paddr;
uint64_t block_count;
uint32_t block_size;
uint32_t blocks_per_chunk;
uint32_t chunks_per_cib;
uint32_t cibs_per_cab;
uint32_t cib_count;
uint32_t cab_count;
const le_uint64_t *cxb_oid;
uint32_t cab_id;
uint32_t cib_id;
uint32_t chunk_id;
uint32_t blk_id;
size_t n;
int devidx;
uint64_t offs = 0;
std::vector<uint64_t> cib_oid_list;
// Get NX superblock
if (!Read(nx_data, 0, 1))
return false;
bd.DumpNode(nx_data.data(), 0);
if (!VerifyBlock(nx_data.data(), nx_data.size()))
{
std::cerr << "Superblock checksum error" << std::endl;
return false;
}
nx = reinterpret_cast<const nx_superblock_t *>(nx_data.data());
// TODO: Scan xp_desc for most recent nxsb
// Get Check Point Info Block
if (!Read(cpm_data, nx->nx_xp_desc_base + nx->nx_xp_desc_index, 1))
{
std::cerr << "Error reading the cpm block" << std::endl;
return false;
}
bd.DumpNode(cpm_data.data(), cpm_data.size());
if (!VerifyBlock(cpm_data.data(), cpm_data.size()))
return false;
cpm = reinterpret_cast<const checkpoint_map_phys_t *>(cpm_data.data());
cm = cpm_lookup(cpm, nx->nx_spaceman_oid);
if (cm == 0)
return false;
// Get Spaceman
if (!Read(sm_data, cm->cpm_paddr, cm->cpm_size / nx->nx_block_size))
return false;
bd.DumpNode(sm_data.data(), cm->cpm_paddr);
if (!VerifyBlock(sm_data.data(), sm_data.size()))
return false;
sm = reinterpret_cast<const spaceman_phys_t *>(sm_data.data());
os << "Now dumping blocks according to bitmap ..." << std::endl;
block_size = sm->sm_block_size;
blocks_per_chunk = sm->sm_blocks_per_chunk;
chunks_per_cib = sm->sm_chunks_per_cib;
cibs_per_cab = sm->sm_cibs_per_cab;
for (devidx = SD_MAIN; devidx < SD_COUNT; devidx++)
{
std::cout << "Dumping device " << devidx << std::endl;
if (devidx == SD_TIER2 && m_dev_tier2 == 0)
{
std::cout << "Aborting" << std::endl;
break;
}
offs = (devidx == SD_TIER2) ? (FUSION_TIER2_DEVICE_BYTE_ADDR / m_blocksize) : 0;
block_count = sm->sm_dev[devidx].sm_block_count;
cib_count = sm->sm_dev[devidx].sm_cib_count;
cab_count = sm->sm_dev[devidx].sm_cab_count;
cxb_oid = reinterpret_cast<const le_uint64_t *>(sm_data.data() + sm->sm_dev[devidx].sm_addr_offset);
paddr = 0;
std::cout << std::hex;
#if 0
static const uint8_t vek[0x20] = { 0 /* enter VEK here */ };
m_is_encrypted = true;
m_aes.SetKey(vek, vek + 0x10);
#endif
cib_oid_list.clear();
cib_oid_list.reserve(cib_count);
if (cab_count > 0)
{
for (cab_id = 0; cab_id < cab_count; cab_id++)
{
if (!Read(cxb_data, cxb_oid[cab_id], 1))
return false;
if (!VerifyBlock(cxb_data.data(), cxb_data.size()))
return false;
cab = reinterpret_cast<const cib_addr_block_t *>(cxb_data.data());
for (n = 0; n < cab->cab_cib_count; n++)
cib_oid_list.push_back(cab->cab_cib_addr[n]);
}
}
else
{
for (n = 0; n < cib_count; n++)
cib_oid_list.push_back(cxb_oid[n]);
}
for (cib_id = 0; cib_id < cib_count; cib_id++)
{
std::cout << "cib " << cib_id << std::endl;
if (!Read(cxb_data, cib_oid_list[cib_id], 1))
return false;
if (!VerifyBlock(cxb_data.data(), cxb_data.size()))
return false;
cib = reinterpret_cast<const chunk_info_block_t *>(cxb_data.data());
for (chunk_id = 0; chunk_id < cib->cib_chunk_info_count; chunk_id++)
{
if (g_abort)
return false;
std::cout << " chunk " << chunk_id << " avail=" << cib->cib_chunk_info[chunk_id].ci_free_count << " paddr=" << paddr << std::endl;
if (cib->cib_chunk_info[chunk_id].ci_bitmap_addr == 0)
{
paddr += blocks_per_chunk;
continue;
}
if (!Read(bmp_data, cib->cib_chunk_info[chunk_id].ci_bitmap_addr, 1))
return false;
for (blk_id = 0; blk_id < blocks_per_chunk && paddr < block_count; blk_id++)
{
if (bmp_data[blk_id >> 3] & (1 << (blk_id & 7)))
{
Read(blk_data, paddr + offs, 1);
if (VerifyBlock(blk_data.data(), m_blocksize))
bd.DumpNode(blk_data.data(), paddr + offs);
else if (m_is_encrypted)
{
Decrypt(blk_data.data(), blk_data.size(), paddr + offs);
if (VerifyBlock(blk_data.data(), m_blocksize))
bd.DumpNode(blk_data.data(), paddr + offs);
}
}
++paddr;
}
}
}
}
return true;
}
bool Dumper::DumpBlockList(std::ostream& os)
{
using namespace std;
constexpr size_t BLOCKSIZE = 0x1000; // Should make this dynamic ...
uint64_t bid;
uint8_t block[BLOCKSIZE];
const obj_phys_t * const o = reinterpret_cast<const obj_phys_t *>(block);
const btree_node_phys_t * const bt = reinterpret_cast<const btree_node_phys_t *>(block);
bool last_was_used = false;
os << hex << uppercase << setfill('0');
os << "[Block] | oid | xid | type | subtype | Page | Levl | Entries | Description" << endl;
os << "---------+----------+----------+----------+----------+------+------+----------+---------------------------------" << endl;
for (bid = 0; bid < (m_size_main / BLOCKSIZE) && !g_abort; bid++)
{
if ((bid & 0xFFF) == 0)
{
std::cout << '.';
std::cout.flush();
}
Read(block, bid, 1);
if (IsEmptyBlock(block, BLOCKSIZE))
{
if (last_was_used)
os << "---------+----------+----------+----------+----------+------+------+----------+ Empty" << endl;
last_was_used = false;
continue;
}
if (VerifyBlock(block, BLOCKSIZE))
{
os << setw(8) << bid << " | ";
os << setw(8) << o->o_oid << " | ";
os << setw(8) << o->o_xid << " | ";
os << setw(8) << o->o_type << " | ";
os << setw(8) << o->o_subtype << " | ";
os << setw(4) << bt->btn_flags << " | ";
os << setw(4) << bt->btn_level << " | ";
os << setw(8) << bt->btn_nkeys << " | ";
os << BlockDumper::GetNodeType(o->o_type, o->o_subtype);
if ((o->o_type & OBJECT_TYPE_MASK) == OBJECT_TYPE_BTREE)
os << " [Root]";
os << endl;
last_was_used = true;
}
else
{
os << setw(8) << bid;
os << " | | | | | | | | Data" << endl;
last_was_used = true;
}
}
if (m_dev_tier2)
{
const uint64_t off = FUSION_TIER2_DEVICE_BYTE_ADDR / m_blocksize;
for (bid = 0; bid < (m_size_tier2 / BLOCKSIZE) && !g_abort; bid++)
{
if ((bid & 0xFFF) == 0)
{
std::cout << '.';
std::cout.flush();
}
Read(block + off, bid, 1);
if (IsEmptyBlock(block, BLOCKSIZE))
{
if (last_was_used)
os << "---------+----------+----------+----------+----------+------+------+----------+ Empty" << endl;
last_was_used = false;
continue;
}
if (VerifyBlock(block, BLOCKSIZE))
{
os << setw(8) << bid << " | ";
os << setw(8) << o->o_oid << " | ";
os << setw(8) << o->o_xid << " | ";
os << setw(8) << o->o_type << " | ";
os << setw(8) << o->o_subtype << " | ";
os << setw(4) << bt->btn_flags << " | ";
os << setw(4) << bt->btn_level << " | ";
os << setw(8) << bt->btn_nkeys << " | ";
os << BlockDumper::GetNodeType(o->o_type, o->o_subtype);
if ((o->o_type & OBJECT_TYPE_MASK) == OBJECT_TYPE_BTREE)
os << " [Root]";
os << endl;
last_was_used = true;
}
else
{
os << setw(8) << bid;
os << " | | | | | | | | Data" << endl;
last_was_used = true;
}
}
}
os << endl;
return true;
}
bool Dumper::Read(void* data, uint64_t paddr, uint64_t cnt)
{
uint64_t offs;
uint64_t size;
offs = paddr * m_blocksize;
size = cnt * m_blocksize;
if (offs & FUSION_TIER2_DEVICE_BYTE_ADDR)
{
offs = offs - FUSION_TIER2_DEVICE_BYTE_ADDR + m_base_tier2;
return m_dev_tier2->Read(data, offs, size);
}
else
{
offs = offs + m_base_main;
return m_dev_main->Read(data, offs, size);
}
}
bool Dumper::Read(std::vector<uint8_t>& data, uint64_t paddr, uint64_t cnt)
{
if (data.size() != cnt * m_blocksize)
data.resize(cnt * m_blocksize);
return Read(data.data(), paddr, cnt);
}
void Dumper::Decrypt(uint8_t* data, size_t size, uint64_t paddr)
{
uint64_t cs_factor = m_blocksize / 0x200;
uint64_t uno = paddr * cs_factor;
size_t k;
for (k = 0; k < size; k += 0x200)
{
m_aes.Decrypt(data + k, data + k, 0x200, uno);
uno++;
}
}
const checkpoint_mapping_t *Dumper::cpm_lookup(const checkpoint_map_phys_t* cpm, uint64_t oid)
{
uint32_t k;
uint32_t cnt;
cnt = cpm->cpm_count;
for (k = 0; k < cnt; k++)
{
if (cpm->cpm_map[k].cpm_oid == oid)
return &cpm->cpm_map[k];
}
return nullptr;
}
+40
查看文件
@@ -0,0 +1,40 @@
#pragma once
#include <vector>
#include <iosfwd>
#include <Crypto/AesXts.h>
extern volatile bool g_abort; // In Apfs.cpp
class Device;
class Dumper
{
public:
Dumper(Device *dev_main, Device *dev_tier2);
~Dumper();
bool Initialize();
bool DumpContainer(std::ostream &os);
bool DumpBlockList(std::ostream &os);
private:
bool Read(void *data, uint64_t paddr, uint64_t cnt);
bool Read(std::vector<uint8_t> &data, uint64_t paddr, uint64_t cnt);
void Decrypt(uint8_t *data, size_t size, uint64_t paddr);
const checkpoint_mapping_t* cpm_lookup(const checkpoint_map_phys_t *cpm, uint64_t oid);
Device *m_dev_main;
Device *m_dev_tier2;
uint64_t m_base_main;
uint64_t m_size_main;
uint64_t m_base_tier2;
uint64_t m_size_tier2;
uint32_t m_blocksize;
AesXts m_aes;
bool m_is_encrypted;
};
+202
查看文件
@@ -0,0 +1,202 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <memory>
#include <cstring>
#include <iostream>
#include <fstream>
#include <ApfsLib/Device.h>
#include <ApfsLib/ApfsContainer.h>
#include <ApfsLib/ApfsVolume.h>
#include <ApfsLib/BlockDumper.h>
#include <ApfsLib/ApfsDir.h>
#include <ApfsLib/GptPartitionMap.h>
int main(int argc, char *argv[])
{
bool rc;
int volume_id;
int n;
std::unique_ptr<Device> main_disk;
std::unique_ptr<Device> tier2_disk;
uint64_t main_offset;
uint64_t tier2_offset;
uint64_t main_size;
uint64_t tier2_size;
const char *main_name;
const char *tier2_name;
const char *output_name;
if (argc < 3)
{
std::cerr << "Syntax: apfs-dump-quick <main-device> [-f fusion-device] <Logfile.txt>" << std::endl;
return -1;
}
main_name = argv[1];
if (!strcmp(argv[2], "-f"))
{
if (argc < 5)
std::cerr << "Syntax: apfs-dump-quick <main-device> [-f fusion-secondary-device] <Logfile.txt>" << std::endl;
tier2_name = argv[3];
output_name = argv[4];
}
else
{
tier2_name = nullptr;
output_name = argv[2];
}
g_debug = 255;
main_disk.reset(Device::OpenDevice(main_name));
if (tier2_name)
tier2_disk.reset(Device::OpenDevice(tier2_name));
if (!main_disk)
{
std::cerr << "Unable to open device " << main_name << std::endl;
return -1;
}
if (tier2_name && !tier2_disk)
{
std::cerr << "Unable to open secondary device " << tier2_name << std::endl;
return -1;
}
std::ofstream st;
st.open(output_name);
if (!st.is_open())
{
std::cerr << "Unable to open output file " << output_name << std::endl;
main_disk->Close();
return -1;
}
main_offset = 0;
main_size = main_disk->GetSize();
tier2_offset = 0;
if (tier2_disk)
tier2_size = tier2_disk->GetSize();
else
tier2_size = 0;
GptPartitionMap gpt;
if (gpt.LoadAndVerify(*main_disk.get()))
{
std::cout << "Info: Found valid GPT partition table on main device. Dumping first APFS partition." << std::endl;
n = gpt.FindFirstAPFSPartition();
if (n != -1)
gpt.GetPartitionOffsetAndSize(n, main_offset, main_size);
}
if (tier2_disk && gpt.LoadAndVerify(*tier2_disk.get()))
{
std::cout << "Info: Found valid GPT partition table on tier2 device. Dumping first APFS partition." << std::endl;
n = gpt.FindFirstAPFSPartition();
if (n != -1)
gpt.GetPartitionOffsetAndSize(n, tier2_offset, tier2_size);
}
std::unique_ptr<ApfsContainer> container(new ApfsContainer(main_disk.get(), main_offset, main_size, tier2_disk.get(), tier2_offset, tier2_size));
rc = container->Init();
if (!rc)
{
std::cerr << "Unable to init container." << std::endl;
container.reset();
main_disk->Close();
return -1;
}
#if 1
BlockDumper bd(st, container->GetBlocksize());
container->dump(bd);
#if 1
for (volume_id = 0; volume_id < NX_MAX_FILE_SYSTEMS; volume_id++)
{
ApfsVolume *vol;
vol = container->GetVolume(volume_id);
if (vol)
{
std::cout << "Volume " << volume_id << ": " << vol->name() << std::endl;
vol->dump(bd);
delete vol;
}
}
#endif
#endif
#if 0
ApfsVolume *vol;
vol = container->GetVolume(0);
if (vol)
{
ApfsDir dir(*vol);
ApfsDir::Inode ino;
std::vector<ApfsDir::Name> root_list;
#if 0
dir.GetInode(ino, 0x15);
std::cout << "Directory is called " << ino.name << std::endl;
dir.ListDirectory(root_list, 0x15);
for (auto it = root_list.cbegin(); it != root_list.cend(); ++it)
{
std::cout << it->name << " : " << it->inode_id << std::endl;
}
#endif
#if 1
std::vector<uint8_t> buf;
buf.resize(0x29000);
dir.ReadFile(buf.data(), 0x39, 0, 0x29000);
#endif
}
#endif
container.reset();
st.close();
main_disk->Close();
return 0;
}
+482
查看文件
@@ -0,0 +1,482 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <cstring>
#include <iostream>
#include <fstream>
#include "ApfsContainer.h"
#include "ApfsVolume.h"
#include "Util.h"
#include "BlockDumper.h"
#include "Global.h"
int g_debug = 0;
bool g_lax = false;
ApfsContainer::ApfsContainer(Device *disk_main, uint64_t main_start, uint64_t main_len, Device *disk_tier2, uint64_t tier2_start, uint64_t tier2_len) :
m_main_disk(disk_main),
m_main_part_start(main_start),
m_main_part_len(main_len),
m_tier2_disk(disk_tier2),
m_tier2_part_start(tier2_start),
m_tier2_part_len(tier2_len),
m_cpm(*this),
m_omap(*this),
// m_omap_tree(*this),
m_fq_tree_mgr(*this),
m_fq_tree_vol(*this),
m_keymgr(*this)
{
m_sm = nullptr;
}
ApfsContainer::~ApfsContainer()
{
}
bool ApfsContainer::Init(xid_t req_xid)
{
std::vector<uint8_t> blk;
blk.resize(0x1000);
if (!m_main_disk->Read(blk.data(), m_main_part_start, 0x1000))
{
std::cerr << "Reading block 0 from main device failed." << std::endl;
return false;
}
memcpy(&m_nx, blk.data(), sizeof(nx_superblock_t));
if (m_nx.nx_magic != NX_MAGIC)
{
std::cerr << "This doesn't seem to be an apfs volume (invalid superblock)." << std::endl;
return false;
}
if (m_nx.nx_block_size != 0x1000)
{
blk.resize(m_nx.nx_block_size);
m_main_disk->Read(blk.data(), m_main_part_start, blk.size());
}
if (!VerifyBlock(blk.data(), blk.size()))
return false;
memcpy(&m_nx, blk.data(), sizeof(nx_superblock_t));
// Scan container for most recent superblock (might fix segfaults)
uint64_t max_xid = 0;
paddr_t max_paddr = 0;
paddr_t paddr;
std::vector<uint8_t> tmp;
tmp.resize(m_nx.nx_block_size);
for (paddr = m_nx.nx_xp_desc_base; paddr < (m_nx.nx_xp_desc_base + m_nx.nx_xp_desc_blocks); paddr++)
{
if (!ReadBlocks(tmp.data(), paddr, 1))
return false;
if (!VerifyBlock(tmp.data(), tmp.size()))
continue;
const nx_superblock_t *sb = reinterpret_cast<const nx_superblock_t *>(tmp.data());
if ((sb->nx_o.o_type & OBJECT_TYPE_MASK) != OBJECT_TYPE_NX_SUPERBLOCK)
continue;
if (req_xid) {
if (req_xid == sb->nx_o.o_xid) {
max_xid = req_xid;
max_paddr = paddr;
break;
}
} else {
if (sb->nx_o.o_xid > max_xid)
{
max_xid = sb->nx_o.o_xid;
max_paddr = paddr;
}
}
}
if (max_paddr)
{
// if (g_debug & Dbg_Errors)
// std::cout << "Found more recent xid " << max_xid << " than superblock 0 contained (" << m_nx.nx_o.o_xid << ")." << std::endl;
if (g_debug & Dbg_Info)
std::cout << "Mounting xid different from NXSB at 0 (xid = " << m_nx.nx_o.o_xid << "). xid = " << max_xid << std::endl;
ReadBlocks(tmp.data(), max_paddr, 1);
memcpy(&m_nx, tmp.data(), sizeof(nx_superblock_t));
}
if (g_debug & Dbg_Info)
std::cout << "Mounting xid " << m_nx.nx_o.o_xid << std::endl;
if ((m_nx.nx_incompatible_features & NX_INCOMPAT_FUSION) && !m_tier2_disk)
{
std::cerr << "Need to specify two devices for a fusion drive." << std::endl;
return false;
}
if (!m_cpm.Init(m_nx.nx_xp_desc_base + m_nx.nx_xp_desc_index, m_nx.nx_xp_desc_len - 1))
{
std::cerr << "Failed to load checkpoint map" << std::endl;
return false;
}
if (!m_omap.Init(m_nx.nx_omap_oid, m_nx.nx_o.o_xid))
{
std::cerr << "Failed to load nx omap" << std::endl;
return false;
}
omap_res_t omr;
if (!m_cpm.Lookup(omr, m_nx.nx_spaceman_oid, m_nx.nx_o.o_xid))
{
std::cerr << "Failed to map spaceman oid" << std::endl;
return false;
}
m_sm_data.resize(omr.size);
ReadBlocks(m_sm_data.data(), omr.paddr, omr.size / GetBlocksize());
m_sm = reinterpret_cast<const spaceman_phys_t *>(m_sm_data.data());
if (!VerifyBlock(m_sm_data.data(), m_sm_data.size())) {
std::cerr << "Checksum error in spaceman" << std::endl;
return false;
}
if ((m_sm->sm_o.o_type & OBJECT_TYPE_MASK) != OBJECT_TYPE_SPACEMAN)
{
std::cerr << "Spaceman has wrong type " << m_sm->sm_o.o_type << std::endl;
return false;
}
if (m_sm->sm_fq[SFQ_IP].sfq_tree_oid != 0)
m_fq_tree_mgr.Init(m_sm->sm_fq[SFQ_IP].sfq_tree_oid, m_sm->sm_o.o_xid, &m_cpm);
if (m_sm->sm_fq[SFQ_MAIN].sfq_tree_oid != 0)
m_fq_tree_vol.Init(m_sm->sm_fq[SFQ_MAIN].sfq_tree_oid, m_sm->sm_o.o_xid, &m_cpm);
// m_omap_tree.Init(m_nx.nx_omap_oid, m_nx.hdr.o_xid, nullptr);
// m_sb.nx_spaceman_oid
if ((m_nx.nx_keylocker.pr_start_addr != 0) && (m_nx.nx_keylocker.pr_block_count != 0))
{
if (!m_keymgr.Init(m_nx.nx_keylocker.pr_start_addr, m_nx.nx_keylocker.pr_block_count, m_nx.nx_uuid))
{
std::cerr << "Initialization of KeyManager failed." << std::endl;
return false;
}
}
return true;
}
ApfsVolume *ApfsContainer::GetVolume(unsigned int fsid, const std::string &passphrase, xid_t snap_xid)
{
ApfsVolume *vol = nullptr;
oid_t oid;
omap_res_t omr;
bool rc;
if (fsid >= 100)
return nullptr;
m_passphrase = passphrase;
oid = m_nx.nx_fs_oid[fsid];
if (oid == 0)
return nullptr;
if (!m_omap.Lookup(omr, oid, m_nx.nx_o.o_xid))
return nullptr;
// std::cout << std::hex << "Loading Volume " << index << ", nodeid = " << nodeid << ", version = " << m_sb.hdr.version << ", blkid = " << blkid << std::endl;
if (omr.paddr == 0)
return nullptr;
vol = new ApfsVolume(*this);
if (snap_xid != 0)
rc = vol->MountSnapshot(omr.paddr, snap_xid);
else
rc = vol->Init(omr.paddr);
if (rc == false)
{
delete vol;
vol = nullptr;
}
return vol;
}
bool ApfsContainer::GetVolumeInfo(unsigned int fsid, apfs_superblock_t& apsb)
{
oid_t oid;
omap_res_t omr;
std::vector<uint8_t> apsb_raw;
if (fsid >= NX_MAX_FILE_SYSTEMS)
return false;
oid = m_nx.nx_fs_oid[fsid];
if (oid == 0)
return false;
if (!m_omap.Lookup(omr, oid, m_nx.nx_o.o_xid))
return false;
if (omr.paddr == 0)
return false;
apsb_raw.resize(GetBlocksize());
if (!ReadAndVerifyHeaderBlock(apsb_raw.data(), omr.paddr))
return false;
memcpy(&apsb, apsb_raw.data(), sizeof(apfs_superblock_t));
return true;
}
bool ApfsContainer::ReadBlocks(uint8_t * data, paddr_t paddr, uint64_t blkcnt) const
{
uint64_t offs;
uint64_t size;
//if ((paddr + blkcnt) > m_nx.nx_block_count)
// return false;
offs = m_nx.nx_block_size * paddr;
size = m_nx.nx_block_size * blkcnt;
if (offs & FUSION_TIER2_DEVICE_BYTE_ADDR)
{
if (!m_tier2_disk)
return false;
offs = offs - FUSION_TIER2_DEVICE_BYTE_ADDR + m_tier2_part_start;
return m_tier2_disk->Read(data, offs, size);
}
else
{
if (!m_main_disk)
return false;
offs = offs + m_main_part_start;
return m_main_disk->Read(data, offs, size);
}
}
bool ApfsContainer::ReadAndVerifyHeaderBlock(uint8_t * data, paddr_t paddr) const
{
if (!ReadBlocks(data, paddr))
return false;
if (!VerifyBlock(data, m_nx.nx_block_size)) {
if (g_debug & Dbg_Errors) {
std::cerr << "ReadAndVerifyHeaderBlock checksum error." << std::endl;
DumpHex(std::cerr, data, m_nx.nx_block_size);
}
return false;
}
return true;
}
bool ApfsContainer::GetVolumeKey(uint8_t *key, const apfs_uuid_t & vol_uuid, const char *password)
{
if (!m_keymgr.IsValid())
return false;
if (password)
{
return m_keymgr.GetVolumeKey(key, vol_uuid, password);
}
else
{
if (m_passphrase.empty())
return false;
return m_keymgr.GetVolumeKey(key, vol_uuid, m_passphrase.c_str());
}
}
bool ApfsContainer::GetPasswordHint(std::string & hint, const apfs_uuid_t & vol_uuid)
{
return m_keymgr.GetPasswordHint(hint, vol_uuid);
}
void ApfsContainer::dump(BlockDumper& bd)
{
std::vector<uint8_t> blk;
paddr_t paddr;
uint32_t index;
uint32_t last_index;
bd.st() << "Dumping Container" << std::endl;
bd.st() << "-----------------" << std::endl;
bd.st() << std::endl;
bd.st() << std::endl;
blk.resize(GetBlocksize());
ReadAndVerifyHeaderBlock(blk.data(), 0);
bd.DumpNode(blk.data(), 0);
#if 1
if (m_keymgr.IsValid())
m_keymgr.dump(bd.st());
#endif
/*
if (m_keybag.size())
bd.DumpNode(m_keybag.data(), m_sb.keybag_blk_start);
*/
#if 0
for (paddr = m_nx.nx_xp_desc_base; paddr < (m_nx.nx_xp_desc_base + m_nx.nx_xp_desc_blocks); paddr++)
{
ReadAndVerifyHeaderBlock(blk.data(), paddr);
bd.DumpNode(blk.data(), paddr);
}
#endif
#if 0
for (paddr = m_nx.nx_xp_data_base; paddr < (m_nx.nx_xp_data_base + m_nx.nx_xp_data_blocks); paddr++)
{
ReadAndVerifyHeaderBlock(blk.data(), paddr);
bd.DumpNode(blk.data(), paddr);
}
#endif
#if 1
bd.st() << std::endl << "Dumping XP desc area (current SB):" << std::endl;
paddr = m_nx.nx_xp_desc_base;
last_index = m_nx.nx_xp_desc_index + m_nx.nx_xp_desc_len;
if (last_index >= m_nx.nx_xp_desc_blocks)
last_index -= m_nx.nx_xp_desc_blocks;
for (index = m_nx.nx_xp_desc_index; index != last_index;)
{
ReadAndVerifyHeaderBlock(blk.data(), paddr + index);
bd.DumpNode(blk.data(), paddr + index);
index++;
if (index >= m_nx.nx_xp_desc_blocks)
index -= m_nx.nx_xp_desc_blocks;
}
bd.st() << std::endl << "Dumping XP data area (current SB):" << std::endl;
paddr = m_nx.nx_xp_data_base;
last_index = m_nx.nx_xp_data_index + m_nx.nx_xp_data_len;
if (last_index >= m_nx.nx_xp_data_blocks)
last_index -= m_nx.nx_xp_data_blocks;
for (index = m_nx.nx_xp_data_index; index != last_index;)
{
ReadAndVerifyHeaderBlock(blk.data(), paddr + index);
bd.DumpNode(blk.data(), paddr + index);
index++;
if (index >= m_nx.nx_xp_data_blocks)
index -= m_nx.nx_xp_data_blocks;
}
#endif
if (m_nx.nx_efi_jumpstart)
{
ReadAndVerifyHeaderBlock(blk.data(), m_nx.nx_efi_jumpstart);
bd.DumpNode(blk.data(), m_nx.nx_efi_jumpstart);
}
ReadAndVerifyHeaderBlock(blk.data(), m_nx.nx_omap_oid);
bd.DumpNode(blk.data(), m_nx.nx_omap_oid);
{
size_t bs = bd.GetBlockSize();
bd.SetBlockSize(m_sm_data.size());
bd.DumpNode(m_sm_data.data(), m_nx.nx_spaceman_oid);
bd.SetBlockSize(bs);
}
uint64_t oid;
size_t k;
for (k = 0; k < m_sm->sm_ip_bm_block_count; k++)
{
oid = m_sm->sm_ip_bm_base + k;
bd.st() << "Dumping IP Bitmap block " << k << std::endl;
ReadBlocks(blk.data(), oid);
bd.DumpNode(blk.data(), oid);
bd.st() << std::endl;
}
m_omap.dump(bd);
// m_omap_tree.dump(bd);
m_fq_tree_mgr.dump(bd);
m_fq_tree_vol.dump(bd);
const le_uint64_t *cxb_oid = reinterpret_cast<const le_uint64_t *>(m_sm_data.data() + m_sm->sm_dev[SD_MAIN].sm_addr_offset);
uint32_t cib_cnt = m_sm->sm_dev[SD_MAIN].sm_cib_count;
uint32_t cab_cnt = m_sm->sm_dev[SD_MAIN].sm_cab_count;
uint32_t cib_id;
uint32_t cab_id;
std::vector<uint64_t> cib_oid_list;
std::vector<uint8_t> cib_data(GetBlocksize());
cib_oid_list.reserve(cib_cnt);
if (cab_cnt != 0)
{
for (cab_id = 0; cab_id < cab_cnt; cab_id++)
{
ReadAndVerifyHeaderBlock(blk.data(), cxb_oid[cab_id]);
bd.DumpNode(blk.data(), cxb_oid[cab_id]);
const cib_addr_block_t *cab = reinterpret_cast<cib_addr_block_t *>(blk.data());
for (cib_id = 0; cib_id < cab->cab_cib_count; cib_id++)
cib_oid_list.push_back(cab->cab_cib_addr[cib_id]);
}
}
else
{
for (cib_id = 0; cib_id < cib_cnt; cib_id++)
cib_oid_list.push_back(cxb_oid[cib_id]);
}
for (cib_id = 0; cib_id < cib_cnt; cib_id++)
{
ReadAndVerifyHeaderBlock(blk.data(), cib_oid_list[cib_id]);
bd.DumpNode(blk.data(), cib_oid_list[cib_id]);
}
}
+85
查看文件
@@ -0,0 +1,85 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "Global.h"
#include "BTree.h"
#include "DiskStruct.h"
#include "Device.h"
#include "CheckPointMap.h"
#include "ApfsNodeMapperBTree.h"
#include "KeyMgmt.h"
#include <cstdint>
#include <vector>
class ApfsVolume;
class BlockDumper;
class ApfsContainer
{
public:
ApfsContainer(Device *disk_main, uint64_t main_start, uint64_t main_len, Device *disk_tier2 = 0, uint64_t tier2_start = 0, uint64_t tier2_len = 0);
~ApfsContainer();
bool Init(xid_t req_xid = 0);
ApfsVolume *GetVolume(unsigned int fsid, const std::string &passphrase = std::string(), xid_t snap_xid = 0);
bool GetVolumeInfo(unsigned int fsid, apfs_superblock_t &apsb);
bool ReadBlocks(uint8_t *data, paddr_t paddr, uint64_t blkcnt = 1) const;
bool ReadAndVerifyHeaderBlock(uint8_t *data, paddr_t paddr) const;
uint32_t GetBlocksize() const { return m_nx.nx_block_size; }
uint64_t GetBlockCount() const { return m_nx.nx_block_count; }
uint64_t GetFreeBlocks() const { return m_sm->sm_dev[SD_MAIN].sm_free_count + m_sm->sm_dev[SD_TIER2].sm_free_count; }
bool GetVolumeKey(uint8_t *key, const apfs_uuid_t &vol_uuid, const char *password = nullptr);
bool GetPasswordHint(std::string &hint, const apfs_uuid_t &vol_uuid);
bool IsUnencrypted() const { return m_keymgr.IsUnencrypted(); }
void dump(BlockDumper& bd);
private:
Device *m_main_disk;
const uint64_t m_main_part_start;
const uint64_t m_main_part_len;
Device *m_tier2_disk;
const uint64_t m_tier2_part_start;
const uint64_t m_tier2_part_len;
std::string m_passphrase;
nx_superblock_t m_nx;
CheckPointMap m_cpm;
ApfsNodeMapperBTree m_omap;
std::vector<uint8_t> m_sm_data;
const spaceman_phys_t *m_sm;
// Block_8_11 -> omap
// BTree m_omap_tree; // see ApfsNodeMapperBTree
BTree m_fq_tree_mgr;
BTree m_fq_tree_vol;
KeyManager m_keymgr;
};
+847
查看文件
@@ -0,0 +1,847 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <iomanip>
#include <algorithm>
#include <cassert>
#include <cstring>
#include "ApfsDir.h"
#include "ApfsVolume.h"
#include "ApfsContainer.h"
#include "BTree.h"
#include "Util.h"
#ifndef _MSC_VER
template<size_t L>
void strcpy_s(char (&dst)[L], const char *src)
{
strncpy(dst, src, L - 1);
dst[L - 1] = 0;
}
#endif
ApfsDir::Inode::Inode()
{
obj_id = 0;
parent_id = 0;
private_id = 0;
create_time = 0;
mod_time = 0;
change_time = 0;
access_time = 0;
internal_flags = 0;
nchildren_nlink = 0;
default_protection_class = 0;
write_generation_counter = 0;
bsd_flags = 0;
owner = 0;
group = 0;
mode = 0;
uncompressed_size = 0;
snap_xid = 0;
delta_tree_oid = 0;
prev_fsize = 0;
ds_size = 0;
ds_alloced_size = 0;
ds_default_crypto_id = 0;
ds_total_bytes_written = 0;
ds_total_bytes_read = 0;
memset(fs_uuid, 0, sizeof(fs_uuid));
sparse_bytes = 0;
document_id = 0;
rdev = 0;
optional_present_flags = 0;
/*
ino = nullptr;
*/
}
ApfsDir::Inode::Inode(const ApfsDir::Inode& o)
{
obj_id = o.obj_id;
parent_id = o.parent_id;
private_id = o.private_id;
create_time = o.create_time;
mod_time = o.mod_time;
change_time = o.change_time;
access_time = o.access_time;
internal_flags = o.internal_flags;
nchildren_nlink = o.nchildren_nlink;
default_protection_class = o.default_protection_class;
write_generation_counter = o.write_generation_counter;
bsd_flags = o.bsd_flags;
owner = o.owner;
group = o.group;
mode = o.mode;
uncompressed_size = o.uncompressed_size;
snap_xid = o.snap_xid;
delta_tree_oid = o.delta_tree_oid;
prev_fsize = o.prev_fsize;
// FinderInfo
ds_size = o.ds_size;
ds_alloced_size = o.ds_alloced_size;
ds_default_crypto_id = o.ds_default_crypto_id;
ds_total_bytes_written = o.ds_total_bytes_written;
ds_total_bytes_read = o.ds_total_bytes_read;
// j_dir_stats_val_t dir_stats;
memcpy(fs_uuid, o.fs_uuid, sizeof(apfs_uuid_t));
sparse_bytes = o.sparse_bytes;
document_id = o.document_id;;
rdev = o.rdev;
name = o.name;
optional_present_flags = o.optional_present_flags;
/*
ino_val_data = o.ino_val_data;
ino = reinterpret_cast<j_inode_val_t *>(ino_val_data.data());
*/
}
ApfsDir::DirRec::DirRec()
{
parent_id = 0;
hash = 0;
file_id = 0;
date_added = 0;
sibling_id = 0;
flags = 0;
has_sibling_id = false;
}
ApfsDir::DirRec::DirRec(const ApfsDir::DirRec& o)
{
parent_id = o.parent_id;
hash = o.hash;
name = o.name;
file_id = o.file_id;
date_added = o.date_added;
sibling_id = o.sibling_id;
flags = o.flags;
has_sibling_id = o.has_sibling_id;
}
ApfsDir::ApfsDir(ApfsVolume &vol) :
m_vol(vol),
m_fs_tree(vol.fstree())
{
m_txt_fmt = vol.getTextFormat();
m_blksize = vol.getContainer().GetBlocksize();
m_blksize_sh = log2(m_blksize);
m_blksize_mask_lo = m_blksize - 1;
m_blksize_mask_hi = ~m_blksize_mask_lo;
m_tmp_blk.resize(m_blksize);
// m_bt.EnableDebugOutput();
}
ApfsDir::~ApfsDir()
{
}
bool ApfsDir::GetInode(ApfsDir::Inode& res, uint64_t inode)
{
BTreeEntry bte;
j_inode_key_t key;
bool rc;
key.hdr.obj_id_and_type = APFS_TYPE_ID(APFS_TYPE_INODE, inode);
rc = m_fs_tree.Lookup(bte, &key, sizeof(j_inode_key_t), CompareStdDirKey, this, true);
if (!rc || (bte.val == nullptr))
return false;
// const uint8_t *idata = reinterpret_cast<const uint8_t *>(bte.val);
const j_inode_val_t *obj = reinterpret_cast<const j_inode_val_t *>(bte.val);
res.obj_id = inode;
res.parent_id = obj->parent_id;
res.private_id = obj->private_id;
res.create_time = obj->create_time;
res.mod_time = obj->mod_time;
res.change_time = obj->change_time;
res.access_time = obj->access_time;
res.internal_flags = obj->internal_flags;
res.nchildren_nlink = obj->nchildren;
res.default_protection_class = obj->default_protection_class;
res.write_generation_counter = obj->write_generation_counter;
res.bsd_flags = obj->bsd_flags;
res.owner = obj->owner;
res.group = obj->group;
res.mode = obj->mode;
res.uncompressed_size = obj->uncompressed_size;
// internal_flags & 0x00200000 => pad2 = uncompressed_size ?
if (bte.val_len > sizeof(j_inode_val_t))
{
const xf_blob_t *xf_hdr = reinterpret_cast<const xf_blob_t *>(obj->xfields);
const x_field_t *xf = reinterpret_cast<const x_field_t *>(xf_hdr->xf_data);
const uint8_t *xdata = obj->xfields + sizeof(xf_blob_t) + xf_hdr->xf_num_exts * sizeof(x_field_t);
uint16_t n;
for (n = 0; n < xf_hdr->xf_num_exts; n++)
{
switch (xf[n].x_type)
{
case INO_EXT_TYPE_SNAP_XID:
assert(xf[n].x_size == sizeof(uint64_t));
res.snap_xid = bswap_le(*reinterpret_cast<const uint64_t *>(xdata));
res.optional_present_flags |= Inode::INO_HAS_SNAP_XID;
break;
case INO_EXT_TYPE_DELTRA_TREE_OID:
assert(xf[n].x_size == sizeof(uint64_t));
res.delta_tree_oid = bswap_le(*reinterpret_cast<const uint64_t *>(xdata));
res.optional_present_flags |= Inode::INO_HAS_DELTA_TREE_OID;
break;
case INO_EXT_TYPE_DOCUMENT_ID:
assert(xf[n].x_size == sizeof(uint32_t));
res.document_id = bswap_le(*reinterpret_cast<const uint32_t *>(xdata));
res.optional_present_flags |= Inode::INO_HAS_DOCUMENT_ID;
break;
case INO_EXT_TYPE_NAME:
res.name.assign(reinterpret_cast<const char *>(xdata), xf[n].x_size);
res.optional_present_flags |= Inode::INO_HAS_NAME;
break;
case INO_EXT_TYPE_PREV_FSIZE:
assert(xf[n].x_size == sizeof(uint64_t));
res.prev_fsize = bswap_le(*reinterpret_cast<const uint64_t *>(xdata));
res.optional_present_flags |= Inode::INO_HAS_PREV_FSIZE;
break;
case INO_EXT_TYPE_FINDER_INFO:
// TODO
break;
case INO_EXT_TYPE_DSTREAM:
{
assert(xf[n].x_size == sizeof(j_dstream_t));
const j_dstream_t *ds = reinterpret_cast<const j_dstream_t *>(xdata);
res.ds_size = ds->size;
res.ds_alloced_size = ds->alloced_size;
res.ds_default_crypto_id = ds->default_crypto_id;
res.ds_total_bytes_written = ds->total_bytes_written;
res.ds_total_bytes_read = ds->total_bytes_read;
res.optional_present_flags |= Inode::INO_HAS_DSTREAM;
}
break;
case INO_EXT_TYPE_DIR_STATS_KEY:
// TODO
break;
case INO_EXT_TYPE_FS_UUID:
assert(xf[n].x_size == sizeof(apfs_uuid_t));
memcpy(res.fs_uuid, xdata, sizeof(apfs_uuid_t));
res.optional_present_flags |= Inode::INO_HAS_FS_UUID;
break;
case INO_EXT_TYPE_SPARSE_BYTES:
assert(xf[n].x_size == sizeof(uint64_t));
res.sparse_bytes = bswap_le(*reinterpret_cast<const uint64_t *>(xdata));
res.optional_present_flags |= Inode::INO_HAS_SPARSE_BYTES;
break;
default:
std::cerr << "Warning: Unknown XF " << xf[n].x_type << " at inode " << inode << std::endl;
break;
}
xdata += ((xf[n].x_size + 7) & ~7);
}
}
return true;
}
bool ApfsDir::ListDirectory(std::vector<DirRec> &dir, uint64_t inode)
{
uint8_t skey_buf[0x500];
BTreeIterator it;
BTreeEntry bte;
bool rc;
uint64_t skey;
const j_key_t *k;
const j_drec_val_t *v;
skey = APFS_TYPE_ID(APFS_TYPE_DIR_REC, inode);
dir.clear();
if (m_txt_fmt & 9)
{
j_drec_hashed_key_t *key = reinterpret_cast<j_drec_hashed_key_t *>(skey_buf);
key->hdr.obj_id_and_type = skey;
key->name_len_and_hash = 0;
key->name[0] = 0;
rc = m_fs_tree.GetIterator(it, key, sizeof(j_drec_hashed_key_t), CompareStdDirKey, this);
}
else
{
j_drec_key_t *key = reinterpret_cast<j_drec_key_t *>(skey_buf);
key->hdr.obj_id_and_type = skey;
key->name_len = 0;
key->name[0] = 0;
rc = m_fs_tree.GetIterator(it, key, sizeof(j_drec_key_t), CompareStdDirKey, this);
}
if (!rc)
return false;
for (;;)
{
DirRec e;
rc = it.GetEntry(bte);
if (!rc)
break;
if (g_debug & Dbg_Dir)
{
DumpBuffer(reinterpret_cast<const uint8_t *>(bte.key), bte.key_len, "entry key");
DumpBuffer(reinterpret_cast<const uint8_t *>(bte.val), bte.val_len, "entry val");
}
k = reinterpret_cast<const j_key_t *>(bte.key);
if (k->obj_id_and_type != skey)
break;
e.parent_id = k->obj_id_and_type & OBJ_ID_MASK;
if (m_txt_fmt != 0)
{
const j_drec_hashed_key_t *hk = reinterpret_cast<const j_drec_hashed_key_t *>(bte.key);
e.hash = hk->name_len_and_hash;
e.name = reinterpret_cast<const char *>(hk->name);
}
else
{
const j_drec_key_t *rk = reinterpret_cast<const j_drec_key_t *>(bte.key);
e.hash = 0;
e.name = reinterpret_cast<const char *>(rk->name);
}
// assert(res.val_len == sizeof(APFS_Name));
v = reinterpret_cast<const j_drec_val_t *>(bte.val);
e.file_id = v->file_id;
e.date_added = v->date_added;
e.flags = v->flags;
if (bte.val_len > sizeof(j_drec_val_t))
{
const xf_blob_t *xf_hdr = reinterpret_cast<const xf_blob_t *>(v->xfields);
const x_field_t *xf = reinterpret_cast<const x_field_t *>(xf_hdr->xf_data);
const uint8_t *xdata = v->xfields + sizeof(xf_blob_t) + xf_hdr->xf_num_exts * sizeof(x_field_t);
uint16_t n;
for (n = 0; n < xf_hdr->xf_num_exts; n++)
{
switch (xf[n].x_type)
{
case DREC_EXT_TYPE_SIBLING_ID:
assert(xf[n].x_size == sizeof(uint64_t));
e.sibling_id = bswap_le(*reinterpret_cast<const uint64_t *>(xdata));
e.has_sibling_id = true;
break;
default:
std::cerr << "Warning: Unknown XF " << xf[n].x_type << " at drec " << e.file_id << std::endl;
break;
}
xdata += ((xf[n].x_size + 7) & ~7);
}
}
dir.push_back(e);
it.next();
}
return true;
}
bool ApfsDir::LookupName(ApfsDir::DirRec& res, uint64_t parent_id, const char* name)
{
bool rc;
BTreeEntry e;
uint8_t srch_key_buf[0x500];
size_t name_len = strlen(name) + 1;
if (name_len > 0x400)
return false;
res.parent_id = parent_id;
res.name = name;
if (m_txt_fmt & 9)
{
j_drec_hashed_key_t *skey = reinterpret_cast<j_drec_hashed_key_t *>(srch_key_buf);
skey->hdr.obj_id_and_type = APFS_TYPE_ID(APFS_TYPE_DIR_REC, parent_id);
skey->name_len_and_hash = HashFilename(reinterpret_cast<const uint8_t *>(name), static_cast<uint16_t>(name_len), (m_txt_fmt & APFS_INCOMPAT_CASE_INSENSITIVE) != 0);
memcpy(skey->name, name, name_len);
if (g_debug & Dbg_Dir)
{
std::cout << "Lookup hashed key: key=" << skey->hdr.obj_id_and_type << " hash=" << skey->name_len_and_hash << " name='" << reinterpret_cast<const char *>(skey->name) << "'" << std::endl;
dump_utf8(std::cout, skey->name);
}
res.hash = skey->name_len_and_hash;
rc = m_fs_tree.Lookup(e, skey, sizeof(j_drec_hashed_key_t) + (skey->name_len_and_hash & J_DREC_LEN_MASK), CompareStdDirKey, this, true);
}
else
{
j_drec_key_t *skey = reinterpret_cast<j_drec_key_t *>(srch_key_buf);
skey->hdr.obj_id_and_type = parent_id | (static_cast<uint64_t>(APFS_TYPE_DIR_REC) << OBJ_TYPE_SHIFT);
skey->name_len = static_cast<uint16_t>(name_len);
memcpy(skey->name, name, name_len);
if (g_debug & Dbg_Dir)
{
std::cout << "Lookup key: key=" << skey->hdr.obj_id_and_type << " name_len=" << skey->name_len << " name='" << reinterpret_cast<const char *>(skey->name) << "'" << std::endl;
dump_utf8(std::cout, skey->name);
}
res.hash = 0;
rc = m_fs_tree.Lookup(e, skey, sizeof(j_drec_key_t) + skey->name_len, CompareStdDirKey, this, true);
}
if (!rc)
{
if (g_debug & Dbg_Dir)
std::cout << "Lookup failed!" << std::endl;
return false;
}
const j_drec_val_t *v = reinterpret_cast<const j_drec_val_t *>(e.val);
res.file_id = v->file_id;
res.date_added = v->date_added;
// v->unk ?
// TODO: SIBLING-ID! XF!
if (g_debug & Dbg_Dir)
std::cout << "Lookup: id = " << res.file_id << std::endl;
return true;
}
bool ApfsDir::ReadFile(void* data, uint64_t inode, uint64_t offs, size_t size)
{
BTreeEntry e;
bool rc;
uint8_t *bdata = reinterpret_cast<uint8_t *>(data);
size_t cur_size;
uint64_t blk_idx;
uint64_t blk_offs;
uint64_t extent_size;
uint64_t extent_offs;
paddr_t extent_paddr;
uint64_t extent_crypto_id;
while (size > 0)
{
if (m_vol.isSealed()) {
fext_tree_key_t key;
const fext_tree_key_t *fext_key = nullptr;
const fext_tree_val_t *fext_val = nullptr;
key.private_id = inode;
key.logical_addr = offs;
rc = m_vol.fexttree().Lookup(e, &key, sizeof(key), CompareFextKey, this, false);
if (!rc) return false;
fext_key = reinterpret_cast<const fext_tree_key_t *>(e.key);
fext_val = reinterpret_cast<const fext_tree_val_t *>(e.val);
extent_offs = offs - fext_key->logical_addr;
extent_size = fext_val->len_and_flags & J_FILE_EXTENT_LEN_MASK;
extent_paddr = fext_val->phys_block_num;
extent_crypto_id = 0; /* TODO: Crypto on sealed volumes? Need later beta for that ... */
} else {
j_file_extent_key_t key;
const j_file_extent_key_t *ext_key = nullptr;
const j_file_extent_val_t *ext_val = nullptr;
key.hdr.obj_id_and_type = APFS_TYPE_ID(APFS_TYPE_FILE_EXTENT, inode);
key.logical_addr = offs;
if (g_debug & Dbg_Dir)
std::cout << "ReadFile(inode=" << inode << ",offs=" << offs << ",size=" << size << ")" << std::endl;
rc = m_fs_tree.Lookup(e, &key, sizeof(key), CompareStdDirKey, this, false);
if (!rc)
return false;
ext_key = reinterpret_cast<const j_file_extent_key_t *>(e.key);
ext_val = reinterpret_cast<const j_file_extent_val_t *>(e.val);
if (g_debug & Dbg_Dir)
{
std::cout << "FileExtent " << ext_key->hdr.obj_id_and_type << " " << ext_key->logical_addr << " => ";
std::cout << ext_val->len_and_flags << " " << ext_val->phys_block_num << " " << ext_val->crypto_id << std::endl;
}
if (ext_key->hdr.obj_id_and_type != key.hdr.obj_id_and_type)
return false;
// Remove flags from length member
extent_offs = offs - ext_key->logical_addr;
extent_size = ext_val->len_and_flags & J_FILE_EXTENT_LEN_MASK;
extent_paddr = ext_val->phys_block_num;
extent_crypto_id = ext_val->crypto_id;
}
blk_idx = extent_offs >> m_blksize_sh;
blk_offs = extent_offs & m_blksize_mask_lo;
cur_size = size;
if ((extent_offs + cur_size) > extent_size)
cur_size = extent_size - extent_offs;
if (cur_size == 0)
break;
if (extent_paddr != 0)
{
if (blk_offs == 0 && cur_size > m_blksize)
cur_size &= m_blksize_mask_hi;
if (blk_offs == 0 && (cur_size & m_blksize_mask_lo) == 0)
{
if (g_debug & Dbg_Dir)
std::cout << "Full read blk " << extent_paddr + blk_idx << " cnt " << (cur_size >> m_blksize_sh) << std::endl;
m_vol.ReadBlocks(bdata, extent_paddr + blk_idx, cur_size >> m_blksize_sh, extent_crypto_id + blk_idx);
}
else
{
if (g_debug & Dbg_Dir)
std::cout << "Partial read blk " << extent_paddr + blk_idx << " cnt 1" << std::endl;
m_vol.ReadBlocks(m_tmp_blk.data(), extent_paddr + blk_idx, 1, extent_crypto_id + blk_idx);
if (blk_offs + cur_size > m_blksize)
cur_size = m_blksize - blk_offs;
if (g_debug & Dbg_Dir)
std::cout << "Partial copy off " << blk_offs << " size " << cur_size << std::endl;
memcpy(bdata, m_tmp_blk.data() + blk_offs, cur_size);
}
}
else
memset(bdata, 0, cur_size);
bdata += cur_size;
offs += cur_size;
size -= cur_size;
// printf("ReadFile: offs=%016lX size=%016lX\n", offs, size);
}
return true;
}
bool ApfsDir::ListAttributes(std::vector<std::string>& names, uint64_t inode)
{
j_inode_key_t skey;
const j_xattr_key_t *ekey;
BTreeIterator it;
BTreeEntry res;
bool rc;
skey.hdr.obj_id_and_type = APFS_TYPE_ID(APFS_TYPE_INODE, inode);
rc = m_fs_tree.GetIterator(it, &skey, sizeof(j_inode_key_t), CompareStdDirKey, this);
if (!rc)
return false;
for (;;)
{
rc = it.GetEntry(res);
if (!rc)
break;
ekey = reinterpret_cast<const j_xattr_key_t *>(res.key);
if ((ekey->hdr.obj_id_and_type & OBJ_ID_MASK) != inode)
break;
if ((ekey->hdr.obj_id_and_type >> OBJ_TYPE_SHIFT) < APFS_TYPE_XATTR)
{
it.next();
continue;
}
if ((ekey->hdr.obj_id_and_type >> OBJ_TYPE_SHIFT) > APFS_TYPE_XATTR)
break;
names.push_back(reinterpret_cast<const char *>(ekey->name));
it.next();
}
return true;
}
bool ApfsDir::GetAttribute(std::vector<uint8_t>& data, uint64_t inode, const char* name)
{
uint8_t skey_buf[0x500];
j_xattr_key_t *skey = reinterpret_cast<j_xattr_key_t *>(skey_buf);
const j_xattr_val_t *attr;
const j_xattr_dstream_t *xstm = nullptr;
// const uint8_t *adata;
BTreeEntry res;
bool rc;
size_t name_len = strlen(name) + 1;
if (name_len > 0x400)
return false;
skey->hdr.obj_id_and_type = APFS_TYPE_ID(APFS_TYPE_XATTR, inode);
skey->name_len = static_cast<int16_t>(name_len);
memcpy(skey->name, name, skey->name_len);
rc = m_fs_tree.Lookup(res, skey, sizeof(j_xattr_key_t) + skey->name_len, CompareStdDirKey, this, true);
if (!rc)
return false;
attr = reinterpret_cast<const j_xattr_val_t *>(res.val);
// adata = reinterpret_cast<const uint8_t *>(res.val) + sizeof(APFS_Xattr_Val);
if (g_debug & Dbg_Dir)
std::cout << "GetAttribute: type=" << attr->flags << std::endl;
if (attr->flags & XATTR_DATA_STREAM)
{
// Attribute contents are stored in a file
assert(attr->xdata_len == sizeof(j_xattr_dstream_t));
xstm = reinterpret_cast<const j_xattr_dstream_t *>(attr->xdata);
if (g_debug & Dbg_Dir)
{
std::cout << "Attribute is link:" << std::endl;
std::cout << " obj_id : " << xstm->xattr_obj_id << std::endl;
std::cout << " size : " << xstm->dstream.size << std::endl;
std::cout << " alloced_size : " << xstm->dstream.alloced_size << std::endl;
std::cout << " default_crypto_id : " << xstm->dstream.default_crypto_id << std::endl;
std::cout << " total_bytes_written : " << xstm->dstream.total_bytes_written << std::endl;
std::cout << " total_bytes_read : " << xstm->dstream.total_bytes_read << std::endl;
}
data.resize(xstm->dstream.alloced_size);
ReadFile(data.data(), xstm->xattr_obj_id, 0, data.size()); // Read must be multiple of 4K ...
data.resize(xstm->dstream.size);
if (g_debug & Dbg_Dir)
{
size_t dmpsize = 0x40;
if (dmpsize > data.size())
dmpsize = data.size();
DumpBuffer(data.data(), dmpsize, "start of attribute content");
}
}
else if (attr->flags & XATTR_DATA_EMBEDDED)
{
// Attribute contents are stored in the btree
data.assign(attr->xdata, attr->xdata + attr->xdata_len);
}
return true;
}
bool ApfsDir::GetAttributeInfo(ApfsDir::XAttr& attr, uint64_t inode, const char* name)
{
uint8_t skey_buf[0x500];
j_xattr_key_t *skey = reinterpret_cast<j_xattr_key_t *>(skey_buf);
const j_xattr_val_t *xv;
const j_xattr_dstream_t *xs = nullptr;
BTreeEntry res;
bool rc;
size_t name_len = strlen(name) + 1;
if (name_len > 0x400)
return false;
skey->hdr.obj_id_and_type = APFS_TYPE_ID(APFS_TYPE_XATTR, inode);
skey->name_len = static_cast<int16_t>(name_len);
memcpy(skey->name, name, skey->name_len);
rc = m_fs_tree.Lookup(res, skey, sizeof(j_xattr_key_t) + skey->name_len, CompareStdDirKey, this, true);
if (!rc)
return false;
xv = reinterpret_cast<const j_xattr_val_t *>(res.val);
attr.flags = xv->flags;
attr.xdata_len = xv->xdata_len;
if (xv->flags & XATTR_DATA_STREAM)
{
xs = reinterpret_cast<const j_xattr_dstream_t *>(xv->xdata);
attr.xstrm = *xs;
}
return true;
}
int ApfsDir::CompareStdDirKey(const void *skey, size_t skey_len, const void *ekey, size_t ekey_len, void *context)
{
// assert(skey_len == 8);
// assert(ekey_len == 8);
(void)ekey_len;
ApfsDir *dir = reinterpret_cast<ApfsDir *>(context);
uint64_t ks = *reinterpret_cast<const le_uint64_t *>(skey);
uint64_t ke = *reinterpret_cast<const le_uint64_t *>(ekey);
// std::cout << std::hex << std::uppercase << std::setfill('0');
ks = (ks << 4) | (ks >> 60);
ke = (ke << 4) | (ke >> 60);
// std::cout << std::setw(16) << ks << " : " << std::setw(16) << ke << std::endl;
if (ke < ks)
return -1;
if (ke > ks)
return 1;
if (skey_len > 8)
{
switch (ks & 0xF)
{
case APFS_TYPE_DIR_REC:
if (dir->m_txt_fmt & 9)
{
const j_drec_hashed_key_t *s = reinterpret_cast<const j_drec_hashed_key_t *>(skey);
const j_drec_hashed_key_t *e = reinterpret_cast<const j_drec_hashed_key_t *>(ekey);
/* TODO: Is this correct? Yes. TODO check _apfs_cstrncmp. */
if ((e->name_len_and_hash & J_DREC_HASH_MASK) < (s->name_len_and_hash & J_DREC_HASH_MASK))
return -1;
if ((e->name_len_and_hash & J_DREC_HASH_MASK) > (s->name_len_and_hash & J_DREC_HASH_MASK))
return 1;
return apfs_strncmp(e->name, e->name_len_and_hash & J_DREC_LEN_MASK, s->name, s->name_len_and_hash & J_DREC_LEN_MASK);
}
else
{
const j_drec_key_t *s = reinterpret_cast<const j_drec_key_t *>(skey);
const j_drec_key_t *e = reinterpret_cast<const j_drec_key_t *>(ekey);
return apfs_strncmp(e->name, e->name_len, s->name, s->name_len);
}
break;
case APFS_TYPE_FILE_EXTENT:
{
const j_file_extent_key_t *s = reinterpret_cast<const j_file_extent_key_t *>(skey);
const j_file_extent_key_t *e = reinterpret_cast<const j_file_extent_key_t *>(ekey);
assert(skey_len == sizeof(j_file_extent_key_t));
assert(ekey_len == sizeof(j_file_extent_key_t));
if (e->logical_addr < s->logical_addr)
return -1;
if (e->logical_addr > s->logical_addr)
return 1;
}
break;
case APFS_TYPE_XATTR:
{
const j_xattr_key_t *s = reinterpret_cast<const j_xattr_key_t *>(skey);
const j_xattr_key_t *e = reinterpret_cast<const j_xattr_key_t *>(ekey);
return apfs_strncmp(e->name, e->name_len, s->name, s->name_len);
}
break;
case APFS_TYPE_FILE_INFO:
{
const j_file_info_key_t *s = reinterpret_cast<const j_file_info_key_t *>(skey);
const j_file_info_key_t *e = reinterpret_cast<const j_file_info_key_t *>(ekey);
if (e->info_and_lba < s->info_and_lba)
return -1;
if (e->info_and_lba > s->info_and_lba)
return 1;
break;
}
}
}
return 0;
}
int ApfsDir::CompareFextKey(const void* skey, size_t skey_len, const void* ekey, size_t ekey_len, void* context)
{
const fext_tree_key_t *s = reinterpret_cast<const fext_tree_key_t *>(skey);
const fext_tree_key_t *e = reinterpret_cast<const fext_tree_key_t *>(ekey);
if (e->private_id < s->private_id)
return -1;
if (e->private_id > s->private_id)
return 1;
if (e->logical_addr < s->logical_addr)
return -1;
if (e->logical_addr > s->logical_addr)
return 1;
return 0;
}
+147
查看文件
@@ -0,0 +1,147 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <string>
#include <vector>
#include "DiskStruct.h"
class BTree;
class ApfsVolume;
class ApfsDir
{
public:
struct Inode
{
Inode();
Inode(const Inode &other);
/* TODO */
uint64_t obj_id;
uint64_t parent_id;
uint64_t private_id;
uint64_t create_time;
uint64_t mod_time;
uint64_t change_time;
uint64_t access_time;
uint64_t internal_flags;
uint64_t nchildren_nlink;
cp_key_class_t default_protection_class;
uint32_t write_generation_counter;
uint32_t bsd_flags;
uint32_t owner;
uint32_t group;
uint16_t mode;
uint64_t uncompressed_size;
uint64_t snap_xid;
uint64_t delta_tree_oid;
uint64_t prev_fsize;
// FinderInfo
uint64_t ds_size;
uint64_t ds_alloced_size;
uint64_t ds_default_crypto_id;
uint64_t ds_total_bytes_written;
uint64_t ds_total_bytes_read;
// j_dir_stats_val_t dir_stats;
apfs_uuid_t fs_uuid;
uint64_t sparse_bytes;
uint32_t document_id;
uint32_t rdev;
std::string name;
uint32_t optional_present_flags;
enum PresentFlags {
INO_HAS_SNAP_XID = 1,
INO_HAS_DELTA_TREE_OID = 2,
INO_HAS_DOCUMENT_ID = 4,
INO_HAS_NAME = 8,
INO_HAS_PREV_FSIZE = 16,
INO_HAS_FINDER_INFO = 64,
INO_HAS_DSTREAM = 128,
INO_HAS_DIR_STATS = 512,
INO_HAS_FS_UUID = 1024,
INO_HAS_SPARSE_BYTES = 4096,
INO_HAS_RDEV = 8192
};
};
struct DirRec
{
DirRec();
DirRec(const DirRec &other);
uint64_t parent_id;
uint32_t hash;
std::string name;
uint64_t file_id;
uint64_t date_added;
uint64_t sibling_id;
uint16_t flags;
bool has_sibling_id;
};
struct XAttr
{
XAttr();
XAttr(const XAttr &other);
uint16_t flags;
uint16_t xdata_len;
j_xattr_dstream_t xstrm;
};
ApfsDir(ApfsVolume &vol);
~ApfsDir();
bool GetInode(Inode &res, uint64_t inode);
bool ListDirectory(std::vector<DirRec> &dir, uint64_t inode);
bool LookupName(DirRec &res, uint64_t parent_id, const char *name);
bool ReadFile(void *data, uint64_t inode, uint64_t offs, size_t size);
bool ListAttributes(std::vector<std::string> &names, uint64_t inode);
bool GetAttribute(std::vector<uint8_t> &data, uint64_t inode, const char *name);
bool GetAttributeInfo(XAttr &attr, uint64_t inode, const char *name);
private:
static int CompareStdDirKey(const void *skey, size_t skey_len, const void *ekey, size_t ekey_len, void *context);
static int CompareFextKey(const void *skey, size_t skey_len, const void *ekey, size_t ekey_len, void *context);
ApfsVolume &m_vol;
BTree &m_fs_tree;
uint32_t m_txt_fmt;
uint32_t m_blksize;
uint64_t m_blksize_mask_hi;
uint64_t m_blksize_mask_lo;
int m_blksize_sh;
std::vector<uint8_t> m_tmp_blk;
};
+29
查看文件
@@ -0,0 +1,29 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ApfsNodeMapper.h"
ApfsNodeMapper::ApfsNodeMapper()
{
}
ApfsNodeMapper::~ApfsNodeMapper()
{
}
+40
查看文件
@@ -0,0 +1,40 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ApfsTypes.h"
struct omap_res_t
{
oid_t oid;
xid_t xid;
uint32_t flags;
uint32_t size;
paddr_t paddr;
};
class ApfsNodeMapper
{
public:
ApfsNodeMapper();
virtual ~ApfsNodeMapper();
virtual bool Lookup(omap_res_t &res, oid_t oid, xid_t xid) = 0;
};
+137
查看文件
@@ -0,0 +1,137 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <cassert>
#include <cstring>
#include <iostream>
#include "ApfsNodeMapperBTree.h"
#include "ApfsContainer.h"
static int CompareOMapKey(const void *skey, size_t skey_len, const void *ekey, size_t ekey_len, void *context)
{
(void)context;
(void)skey_len;
(void)ekey_len;
assert(skey_len == sizeof(omap_key_t));
assert(ekey_len == sizeof(omap_val_t));
const omap_key_t *skey_map = reinterpret_cast<const omap_key_t *>(skey);
const omap_key_t *ekey_map = reinterpret_cast<const omap_key_t *>(ekey);
if (ekey_map->ok_oid < skey_map->ok_oid)
return -1;
if (ekey_map->ok_oid > skey_map->ok_oid)
return 1;
if (ekey_map->ok_xid < skey_map->ok_xid)
return -1;
if (ekey_map->ok_xid > skey_map->ok_xid)
return 1;
return 0;
}
ApfsNodeMapperBTree::ApfsNodeMapperBTree(ApfsContainer &container) :
m_tree(container),
m_container(container)
{
}
ApfsNodeMapperBTree::~ApfsNodeMapperBTree()
{
}
bool ApfsNodeMapperBTree::Init(oid_t omap_oid, xid_t xid)
{
std::vector<uint8_t> blk;
blk.resize(m_container.GetBlocksize());
if (!m_container.ReadAndVerifyHeaderBlock(blk.data(), omap_oid))
{
std::cerr << "ERROR: Invalid omap block @ oid 0x" << std::hex << omap_oid << std::endl;
return false;
}
memcpy(&m_omap, blk.data(), sizeof(omap_phys_t));
if ((m_omap.om_o.o_type & OBJECT_TYPE_MASK) == OBJECT_TYPE_BTREE)
{
std::cerr << "ERROR: Wrong omap type 0x" << std::hex << m_omap.om_o.o_type << std::endl;
return false;
}
return m_tree.Init(m_omap.om_tree_oid, xid);
}
bool ApfsNodeMapperBTree::Lookup(omap_res_t &omr, oid_t oid, xid_t xid)
{
omap_key_t key;
const omap_key_t *res_key = nullptr;
const omap_val_t *res_val = nullptr;
BTreeEntry res;
key.ok_oid = oid;
key.ok_xid = xid;
// std::cout << std::hex << "Omap Lookup: oid = " << oid << ", xid = " << xid << " => ";
if (!m_tree.Lookup(res, &key, sizeof(key), CompareOMapKey, this, false))
{
// std::cout << "NOT FOUND" << std::endl;
std::cerr << std::hex << "oid " << oid << " xid " << xid << " NOT FOUND!!!" << std::endl;
return false;
}
assert(res.val_len == sizeof(omap_val_t));
res_key = reinterpret_cast<const omap_key_t *>(res.key);
res_val = reinterpret_cast<const omap_val_t *>(res.val);
#if 0
if (g_debug & Dbg_Info) {
std::cout << std::hex << "Omap Lookup: oid=" << oid << " xid=" << xid << ": ";
std::cout << "oid=" << res_key->ok_oid << " xid=" << res_key->ok_xid << " => flags=" << res_val->ov_flags << " size=" << res_val->ov_size << " paddr=" << res_val->ov_paddr << std::endl;
}
#endif
if (key.ok_oid != res_key->ok_oid)
{
// std::cout << "NOT FOUND" << std::endl;
std::cerr << std::hex << "oid " << oid << " xid " << xid << " NOT FOUND!!!" << std::endl;
omr.oid = oid;
omr.xid = xid;
omr.flags = 0;
omr.size = 0;
omr.paddr = 0;
return false;
}
// std::cout << val->blockid << std::endl;
omr.oid = res_key->ok_oid;
omr.xid = res_key->ok_xid;
omr.flags = res_val->ov_flags;
omr.size = res_val->ov_size;
omr.paddr = res_val->ov_paddr;
return true;
}
+45
查看文件
@@ -0,0 +1,45 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "DiskStruct.h"
#include "ApfsNodeMapper.h"
#include "BTree.h"
class BlockDumper;
class ApfsNodeMapperBTree : public ApfsNodeMapper
{
public:
ApfsNodeMapperBTree(ApfsContainer &container);
virtual ~ApfsNodeMapperBTree();
bool Init(oid_t omap_oid, xid_t xid);
bool Lookup(omap_res_t & omr, oid_t oid, xid_t xid) override;
void dump(BlockDumper &bd) { m_tree.dump(bd); }
private:
omap_phys_t m_omap;
BTree m_tree;
ApfsContainer &m_container;
};
+29
查看文件
@@ -0,0 +1,29 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <cstdint>
// Commonly used FS types
typedef unsigned char apfs_uuid_t[16];
typedef uint64_t paddr_t; // Apple: int64_t
typedef uint64_t oid_t;
typedef uint64_t xid_t;
+394
查看文件
@@ -0,0 +1,394 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <cstring>
#include <vector>
#include <iostream>
#include "Global.h"
#include "ApfsContainer.h"
#include "ApfsVolume.h"
#include "BlockDumper.h"
#include "Util.h"
ApfsVolume::ApfsVolume(ApfsContainer &container) :
m_container(container),
m_omap(container),
m_fs_tree(container, this),
m_extentref_tree(container, this),
m_snap_meta_tree(container, this),
m_fext_tree(container, this)
{
m_apsb_paddr = 0;
m_is_encrypted = false;
}
ApfsVolume::~ApfsVolume()
{
}
bool ApfsVolume::Init(paddr_t apsb_paddr)
{
std::vector<uint8_t> blk;
m_apsb_paddr = apsb_paddr;
blk.resize(m_container.GetBlocksize());
if (!ReadBlocks(blk.data(), apsb_paddr, 1, 0))
return false;
if (!VerifyBlock(blk.data(), blk.size()))
return false;
memcpy(&m_sb, blk.data(), sizeof(m_sb));
if (m_sb.apfs_magic != APFS_MAGIC)
return false;
if (!m_omap.Init(m_sb.apfs_omap_oid, m_sb.apfs_o.o_xid)) {
std::cerr << "WARNING: Volume omap tree init failed." << std::endl;
return false;
}
if ((m_sb.apfs_fs_flags & 3) != APFS_FS_UNENCRYPTED && !m_container.IsUnencrypted())
{
uint8_t vek[0x20];
std::string str;
std::cout << "Volume " << m_sb.apfs_volname << " is encrypted." << std::endl;
if (!m_container.GetVolumeKey(vek, m_sb.apfs_vol_uuid))
{
if (m_container.GetPasswordHint(str, m_sb.apfs_vol_uuid))
std::cout << "Hint: " << str << std::endl;
std::cout << "Enter Password: ";
GetPassword(str);
if (!m_container.GetVolumeKey(vek, m_sb.apfs_vol_uuid, str.c_str()))
{
std::cout << "Wrong password!" << std::endl;
return false;
}
}
m_aes.SetKey(vek, vek + 0x10);
m_is_encrypted = true;
}
if (!m_fs_tree.Init(m_sb.apfs_root_tree_oid, m_sb.apfs_o.o_xid, &m_omap))
std::cerr << "ERROR: root tree init failed" << std::endl;
if (!m_extentref_tree.Init(m_sb.apfs_extentref_tree_oid, m_sb.apfs_o.o_xid))
std::cerr << "WARNING: extentref tree init failed" << std::endl;
if (!m_snap_meta_tree.Init(m_sb.apfs_snap_meta_tree_oid, m_sb.apfs_o.o_xid))
std::cerr << "WARNING: snap meta tree init failed" << std::endl;
if (m_sb.apfs_incompatible_features & APFS_INCOMPAT_SEALED_VOLUME)
{
if (!m_fext_tree.Init(m_sb.apfs_fext_tree_oid, m_sb.apfs_o.o_xid))
std::cerr << "ERROR: fext tree init failed" << std::endl;
}
return true;
}
bool ApfsVolume::MountSnapshot(paddr_t apsb_paddr, xid_t snap_xid)
{
BTree snap_btree(m_container);
BTreeEntry snap_entry;
j_snap_metadata_key_t snap_key;
const j_snap_metadata_val_t *snap_val = nullptr;
std::vector<uint8_t> blk;
m_apsb_paddr = apsb_paddr;
blk.resize(m_container.GetBlocksize());
if (!ReadBlocks(blk.data(), apsb_paddr, 1, 0))
return false;
if (!VerifyBlock(blk.data(), blk.size()))
return false;
memcpy(&m_sb, blk.data(), sizeof(m_sb));
if (m_sb.apfs_magic != APFS_MAGIC)
return false;
if (m_sb.apfs_snap_meta_tree_oid == 0)
return false;
if (!snap_btree.Init(m_sb.apfs_snap_meta_tree_oid, m_sb.apfs_o.o_xid)) {
std::cerr << "snap meta tree init failed" << std::endl;
return false;
}
snap_key.hdr.obj_id_and_type = APFS_TYPE_ID(APFS_TYPE_SNAP_METADATA, snap_xid);
if (!snap_btree.Lookup(snap_entry, &snap_key, sizeof(snap_key), CompareSnapMetaKey, nullptr, true)) {
std::cerr << "snap xid not found" << std::endl;
return false;
}
snap_val = reinterpret_cast<const j_snap_metadata_val_t *>(snap_entry.val);
if (!m_omap.Init(m_sb.apfs_omap_oid, m_sb.apfs_o.o_xid)) {
std::cerr << "WARNING: Volume omap tree init failed." << std::endl;
return false;
}
if (!ReadBlocks(blk.data(), snap_val->sblock_oid, 1, 0)) {
std::cerr << "failed to read snapshot superblock" << std::endl;
return false;
}
if (!VerifyBlock(blk.data(), blk.size())) {
std::cerr << "snap superblock checksum error" << std::endl;
return false;
}
memcpy(&m_sb, blk.data(), sizeof(m_sb));
if (m_sb.apfs_magic != APFS_MAGIC)
return false;
if ((m_sb.apfs_fs_flags & 3) != APFS_FS_UNENCRYPTED)
{
uint8_t vek[0x20];
std::string str;
std::cout << "Volume " << m_sb.apfs_volname << " is encrypted." << std::endl;
if (!m_container.GetVolumeKey(vek, m_sb.apfs_vol_uuid))
{
if (m_container.GetPasswordHint(str, m_sb.apfs_vol_uuid))
std::cout << "Hint: " << str << std::endl;
std::cout << "Enter Password: ";
GetPassword(str);
if (!m_container.GetVolumeKey(vek, m_sb.apfs_vol_uuid, str.c_str()))
{
std::cout << "Wrong password!" << std::endl;
return false;
}
}
m_aes.SetKey(vek, vek + 0x10);
m_is_encrypted = true;
}
if (!m_fs_tree.Init(m_sb.apfs_root_tree_oid, m_sb.apfs_o.o_xid, &m_omap))
std::cerr << "WARNING: root tree init failed" << std::endl;
if (!m_extentref_tree.Init(m_sb.apfs_extentref_tree_oid, m_sb.apfs_o.o_xid))
std::cerr << "WARNING: extentref tree init failed" << std::endl;
if (!m_snap_meta_tree.Init(m_sb.apfs_snap_meta_tree_oid, m_sb.apfs_o.o_xid))
std::cerr << "WARNING: snap meta tree init failed" << std::endl;
if (m_sb.apfs_incompatible_features & APFS_INCOMPAT_SEALED_VOLUME)
{
if (!m_fext_tree.Init(m_sb.apfs_fext_tree_oid, m_sb.apfs_o.o_xid))
std::cerr << "ERROR: fext tree init failed" << std::endl;
}
return true;
}
void ApfsVolume::dump(BlockDumper& bd)
{
std::vector<uint8_t> blk;
omap_res_t om;
oid_t omap_snapshot_tree_oid = 0;
blk.resize(m_container.GetBlocksize());
if (!ReadBlocks(blk.data(), m_apsb_paddr, 1, 0))
return;
if (!VerifyBlock(blk.data(), blk.size()))
return;
bd.SetTextFlags(m_sb.apfs_incompatible_features & 0xFF);
bd.DumpNode(blk.data(), m_apsb_paddr);
ReadBlocks(blk.data(), m_sb.apfs_omap_oid, 1, 0);
bd.DumpNode(blk.data(), m_sb.apfs_omap_oid);
{
const omap_phys_t *om = reinterpret_cast<const omap_phys_t*>(blk.data());
omap_snapshot_tree_oid = om->om_snapshot_tree_oid;
ReadBlocks(blk.data(), omap_snapshot_tree_oid, 1, 0);
bd.DumpNode(blk.data(), omap_snapshot_tree_oid);
}
if (m_sb.apfs_er_state_oid) {
ReadBlocks(blk.data(), m_sb.apfs_er_state_oid, 1, 0);
bd.DumpNode(blk.data(), m_sb.apfs_er_state_oid);
}
m_omap.dump(bd);
m_fs_tree.dump(bd);
// m_extentref_tree.dump(bd);
m_snap_meta_tree.dump(bd);
if (m_sb.apfs_integrity_meta_oid != 0) {
if (m_omap.Lookup(om, m_sb.apfs_integrity_meta_oid, m_sb.apfs_o.o_xid))
{
ReadBlocks(blk.data(), om.paddr, 1, 0);
bd.DumpNode(blk.data(), om.paddr);
}
}
if (m_sb.apfs_snap_meta_ext_oid != 0) {
if (m_omap.Lookup(om, m_sb.apfs_snap_meta_ext_oid, m_sb.apfs_o.o_xid))
{
ReadBlocks(blk.data(), om.paddr, 1, 0);
bd.DumpNode(blk.data(), om.paddr);
}
}
#if 1
if (m_sb.apfs_fext_tree_oid != 0) {
BTree fxtree(m_container, this);
fxtree.Init(m_sb.apfs_fext_tree_oid, m_sb.apfs_o.o_xid);
fxtree.dump(bd);
}
#endif
BTreeEntry bte;
#if 0
BTreeIterator it;
const j_snap_metadata_key_t *sm_key;
const j_snap_metadata_val_t *sm_val;
if (m_snap_meta_tree.GetIteratorBegin(it)) {
for (;;) {
if (!it.GetEntry(bte)) break;
sm_key = reinterpret_cast<const j_snap_metadata_key_t *>(bte.key);
sm_val = reinterpret_cast<const j_snap_metadata_val_t *>(bte.val);
if ((sm_key->hdr.obj_id_and_type >> OBJ_TYPE_SHIFT) != APFS_TYPE_SNAP_METADATA) break;
ReadBlocks(blk.data(), sm_val->sblock_oid, 1, 0);
bd.DumpNode(blk.data(), sm_val->sblock_oid);
apfs_superblock_t apsb;
memcpy(&apsb, blk.data(), sizeof(apfs_superblock_t));
if (apsb.apfs_omap_oid) {
ReadBlocks(blk.data(), apsb.apfs_omap_oid, 1, 0);
bd.DumpNode(blk.data(), apsb.apfs_omap_oid);
}
if (!it.next()) break;
}
}
bte.clear();
#endif
#if 0
{
BTree omap_tree(m_container, this);
BTreeIterator oit;
omap_phys_t om;
ReadBlocks(blk.data(), m_sb.apfs_omap_oid, 1, 0);
memcpy(&om, blk.data(), sizeof(om));
omap_tree.Init(om.om_tree_oid, om.om_o.o_xid);
if (omap_tree.GetIteratorBegin(oit)) {
const omap_key_t *ok;
const omap_val_t *ov;
for (;;) {
if (!oit.GetEntry(bte)) break;
ok = reinterpret_cast<const omap_key_t*>(bte.key);
ov = reinterpret_cast<const omap_val_t*>(bte.val);
bd.st() << "omap: " << ok->ok_oid << " " << ok->ok_xid << " => " << ov->ov_flags << " " << ov->ov_size << " " << ov->ov_paddr << std::endl;
if (ov->ov_flags & OMAP_VAL_NOHEADER) {
ReadBlocks(blk.data(), ov->ov_paddr, 1, 0);
bd.DumpNode(blk.data(), ov->ov_paddr);
}
if (!oit.next()) break;
}
} else {
bd.st() << "Failed getting omap iterator" << std::endl;
}
}
#endif
}
bool ApfsVolume::ReadBlocks(uint8_t * data, paddr_t paddr, uint64_t blkcnt, uint64_t xts_tweak)
{
constexpr int encryption_block_size = 0x200;
if (!m_container.ReadBlocks(data, paddr, blkcnt))
return false;
if (!m_is_encrypted || (xts_tweak == 0))
return true;
uint64_t cs_factor = m_container.GetBlocksize() / encryption_block_size;
uint64_t uno = xts_tweak * cs_factor;
size_t size = blkcnt * m_container.GetBlocksize();
size_t k;
for (k = 0; k < size; k += encryption_block_size)
{
m_aes.Decrypt(data + k, data + k, encryption_block_size, uno);
uno++;
}
return true;
}
int ApfsVolume::CompareSnapMetaKey(const void* skey, size_t skey_len, const void* ekey, size_t ekey_len, void* context)
{
const j_key_t *ks = reinterpret_cast<const j_key_t*>(skey);
const j_key_t *ke = reinterpret_cast<const j_key_t*>(ekey);
const j_snap_name_key_t *sks;
const j_snap_name_key_t *ske;
if (ke->obj_id_and_type < ks->obj_id_and_type)
return -1;
if (ke->obj_id_and_type > ks->obj_id_and_type)
return 1;
switch (ks->obj_id_and_type >> OBJ_TYPE_SHIFT)
{
case APFS_TYPE_SNAP_METADATA:
break;
case APFS_TYPE_SNAP_NAME:
sks = reinterpret_cast<const j_snap_name_key_t*>(skey);
ske = reinterpret_cast<const j_snap_name_key_t*>(ekey);
return apfs_strncmp(ske->name, ske->name_len, sks->name, sks->name_len);
break;
}
return 0;
}
+71
查看文件
@@ -0,0 +1,71 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <cstdint>
#include "DiskStruct.h"
#include "ApfsNodeMapperBTree.h"
#include "BTree.h"
#include <Crypto/AesXts.h>
class ApfsContainer;
class BlockDumper;
class ApfsVolume
{
public:
ApfsVolume(ApfsContainer &container);
~ApfsVolume();
bool Init(paddr_t apsb_paddr);
bool MountSnapshot(paddr_t apsb_paddr, xid_t snap_xid);
const char *name() const { return reinterpret_cast<const char *>(m_sb.apfs_volname); }
void dump(BlockDumper &bd);
BTree &fstree() { return m_fs_tree; }
BTree &fexttree() { return m_fext_tree; }
uint32_t getTextFormat() const { return m_sb.apfs_incompatible_features & 0x9; }
ApfsContainer &getContainer() const { return m_container; }
bool ReadBlocks(uint8_t *data, paddr_t paddr, uint64_t blkcnt, uint64_t xts_tweak);
bool isSealed() const { return (m_sb.apfs_incompatible_features & APFS_INCOMPAT_SEALED_VOLUME) != 0; }
private:
static int CompareSnapMetaKey(const void *skey, size_t skey_len, const void *ekey, size_t ekey_len, void *context);
ApfsContainer &m_container;
apfs_superblock_t m_sb;
ApfsNodeMapperBTree m_omap;
BTree m_fs_tree;
BTree m_extentref_tree;
BTree m_snap_meta_tree;
BTree m_fext_tree;
paddr_t m_apsb_paddr;
bool m_is_encrypted;
AesXts m_aes;
};
+759
查看文件
@@ -0,0 +1,759 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <cassert>
#include <cstring>
#include <cstdio>
#include <cinttypes>
#include <iostream>
#include <iomanip>
#include "ApfsContainer.h"
#include "ApfsVolume.h"
#include "BTree.h"
#include "Util.h"
#include "BlockDumper.h"
int CompareStdKey(const void *skey, size_t skey_len, const void *ekey, size_t ekey_len, void *context)
{
// assert(skey_len == 8);
// assert(ekey_len == 8);
(void)skey_len;
(void)ekey_len;
(void)context;
uint64_t ks = *reinterpret_cast<const uint64_t *>(skey);
uint64_t ke = *reinterpret_cast<const uint64_t *>(ekey);
if (ke < ks)
return -1;
if (ke > ks)
return 1;
return 0;
}
BTreeEntry::BTreeEntry()
{
key = nullptr;
val = nullptr;
key_len = 0;
val_len = 0;
}
BTreeEntry::~BTreeEntry()
{
}
void BTreeEntry::clear()
{
key = nullptr;
val = nullptr;
key_len = 0;
val_len = 0;
m_node.reset();
}
BTreeNode::BTreeNode(BTree &tree, const uint8_t *block, size_t blocksize, paddr_t paddr, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index) :
m_tree(tree),
m_parent_index(parent_index),
m_parent(parent),
m_paddr(paddr)
{
m_block.assign(block, block + blocksize);
m_btn = reinterpret_cast<const btree_node_phys_t *>(m_block.data());
assert(m_btn->btn_table_space.off == 0);
m_keys_start = sizeof(btree_node_phys_t) + m_btn->btn_table_space.len;
if (m_btn->btn_flags & BTNODE_ROOT)
m_vals_start = blocksize - sizeof(btree_info_t);
else
m_vals_start = blocksize;
}
std::shared_ptr<BTreeNode> BTreeNode::CreateNode(BTree & tree, const uint8_t * block, size_t blocksize, paddr_t paddr, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index)
{
const btree_node_phys_t *btn = reinterpret_cast<const btree_node_phys_t *>(block);
if (btn->btn_flags & BTNODE_FIXED_KV_SIZE)
return std::make_shared<BTreeNodeFix>(tree, block, blocksize, paddr, parent, parent_index);
else
return std::make_shared<BTreeNodeVar>(tree, block, blocksize, paddr, parent, parent_index);
}
BTreeNode::~BTreeNode()
{
}
BTreeNodeFix::BTreeNodeFix(BTree &tree, const uint8_t *block, size_t blocksize, paddr_t paddr, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index) :
BTreeNode(tree, block, blocksize, paddr, parent, parent_index)
{
m_entries = reinterpret_cast<const kvoff_t *>(m_block.data() + sizeof(btree_node_phys_t));
}
bool BTreeNodeFix::GetEntry(BTreeEntry & result, uint32_t index) const
{
result.clear();
if (index >= m_btn->btn_nkeys)
return false;
result.key = m_block.data() + m_keys_start + m_entries[index].k;
result.key_len = m_tree.GetKeyLen();
if (m_entries[index].v != BTOFF_INVALID)
{
result.val = m_block.data() + m_vals_start - m_entries[index].v;
result.val_len = (m_btn->btn_flags & BTNODE_LEAF) ? m_tree.GetValLen() : sizeof(oid_t);
}
else
{
result.val = nullptr;
result.val_len = 0;
}
return true;
}
BTreeNodeVar::BTreeNodeVar(BTree &tree, const uint8_t *block, size_t blocksize, paddr_t paddr, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index) :
BTreeNode(tree, block, blocksize, paddr, parent, parent_index)
{
m_entries = reinterpret_cast<const kvloc_t *>(m_block.data() + sizeof(btree_node_phys_t));
}
bool BTreeNodeVar::GetEntry(BTreeEntry & result, uint32_t index) const
{
result.clear();
if (index >= m_btn->btn_nkeys)
return false;
result.key = m_block.data() + m_keys_start + m_entries[index].k.off;
result.key_len = m_entries[index].k.len;
if (m_entries[index].v.off != BTOFF_INVALID)
{
result.val = m_block.data() + m_vals_start - m_entries[index].v.off;
result.val_len = m_entries[index].v.len;
}
else
{
result.val = nullptr;
result.val_len = 0;
}
return true;
}
BTree::BTree(ApfsContainer &container, ApfsVolume *volume) :
m_container(container)
{
m_volume = volume;
m_root_node = nullptr;
m_omap = nullptr;
m_xid = 0;
m_debug = false;
}
BTree::~BTree()
{
#ifdef BTREE_USE_MAP
m_nodes.clear();
#endif
}
bool BTree::Init(oid_t oid_root, xid_t xid, ApfsNodeMapper *omap)
{
std::shared_ptr<BTreeNode> dummy;
m_omap = omap;
m_oid = oid_root;
m_xid = xid;
if (oid_root == 0) return false;
m_root_node = GetNode(oid_root, dummy, 0);
if (m_root_node)
{
memcpy(&m_treeinfo, m_root_node->block().data() + m_root_node->block().size() - sizeof(btree_info_t), sizeof(btree_info_t));
return true;
}
else
{
std::cerr << "ERROR: BTree Init: Unable to get root node " << oid_root << std::endl;
return false;
}
}
bool BTree::Lookup(BTreeEntry &result, const void *key, size_t key_size, BTCompareFunc func, void *context, bool exact)
{
if (!m_root_node)
return false;
oid_t oid;
oid_t oid_parent;
int index;
std::shared_ptr<BTreeNode> node(m_root_node);
BTreeEntry e;
if (m_debug)
{
// std::cout << std::hex << "BTree::Lookup: key=" << *reinterpret_cast<const uint64_t *>(key) << " root=" << node->nodeid() << std::endl;
std::cout << "BTree::Lookup: ";
DumpHex(std::cout, reinterpret_cast<const uint8_t *>(key), key_size, key_size);
}
while (node->level() > 0)
{
index = FindBin(node, key, key_size, func, context, FindMode::LE);
if (index < 0)
return false;
node->GetEntry(e, index);
// DumpHex(std::cout, reinterpret_cast<const uint8_t*>(e.val), e.val_len, 32);
oid_parent = node->nodeid();
if (node->flags() & BTNODE_HASHED) {
const btn_index_node_val_t *binv = reinterpret_cast<const btn_index_node_val_t*>(e.val);
oid = binv->binv_child_oid + m_oid;
} else {
assert(e.val_len == sizeof(oid_t));
oid = *reinterpret_cast<const oid_t *>(e.val);
}
node = GetNode(oid, node, index);
if (!node)
{
std::cerr << "BTree::Lookup: Node " << oid << " with parent " << oid_parent << " not found." << std::endl;
return false;
}
}
index = FindBin(node, key, key_size, func, context, exact ? FindMode::EQ : FindMode::LE);
if (m_debug)
std::cout << "Result = " << node->nodeid() << ":" << index << std::endl;
if (index < 0)
return false;
node->GetEntry(result, index);
result.m_node = node;
return true;
}
bool BTree::GetIterator(BTreeIterator& it, const void* key, size_t key_size, BTCompareFunc func, void *context)
{
oid_t oid;
int index;
std::shared_ptr<BTreeNode> node(m_root_node);
BTreeEntry e;
if (m_debug)
std::cout << std::hex << "BTree::GetIterator: key=" << *reinterpret_cast<const uint64_t *>(key) << " root=" << node->nodeid() << std::endl;
while (node->level() > 0)
{
index = FindBin(node, key, key_size, func, context, FindMode::LE);
if (index < 0)
index = 0;
node->GetEntry(e, index);
if (node->flags() & BTNODE_HASHED) {
const btn_index_node_val_t *binv = reinterpret_cast<const btn_index_node_val_t*>(e.val);
oid = binv->binv_child_oid + m_oid;
} else {
assert(e.val_len == sizeof(oid_t));
oid = *reinterpret_cast<const oid_t *>(e.val);
}
node = GetNode(oid, node, index);
}
index = FindBin(node, key, key_size, func, context, FindMode::GE);
if (m_debug)
std::cout << "Result = " << node->nodeid() << ":" << index << std::endl;
if (index < 0)
{
index = node->entries_cnt() - 1;
it.Setup(this, node, index);
it.next();
if (m_debug)
std::cout << "Iterator next entry" << std::endl;
}
else
{
it.Setup(this, node, index);
}
#if 0
it.GetEntry(e);
if (func(key, key_size, e.key, e.key_len) < 0)
it.next();
#endif
return true;
}
bool BTree::GetIteratorBegin(BTreeIterator& it)
{
oid_t oid;
std::shared_ptr<BTreeNode> node(m_root_node);
BTreeEntry e;
while (node->level() > 0)
{
if (!node->GetEntry(e, 0))
return false;
if (node->flags() & BTNODE_HASHED) {
const btn_index_node_val_t *binv = reinterpret_cast<const btn_index_node_val_t*>(e.val);
oid = binv->binv_child_oid + m_oid;
} else {
oid = *reinterpret_cast<const oid_t *>(e.val);
}
node = GetNode(oid, node, 0);
}
it.Setup(this, node, 0);
return true;
}
void BTree::dump(BlockDumper& out)
{
if (m_root_node)
DumpTreeInternal(out, m_root_node);
}
void BTree::DumpTreeInternal(BlockDumper& out, const std::shared_ptr<BTreeNode> &node)
{
size_t k;
size_t cnt;
BTreeEntry e;
std::shared_ptr<BTreeNode> child;
oid_t oid_child;
oid_t oid_parent;
if (!node)
return;
out.DumpNode(node->block().data(), node->paddr());
if (node->level() > 0)
{
cnt = node->entries_cnt();
for (k = 0; k < cnt; k++)
{
node->GetEntry(e, k);
/*
if (e.val_len != sizeof(uint64_t))
continue; // assert ...
*/
oid_parent = node->nodeid();
if (node->flags() & BTNODE_HASHED) {
const btn_index_node_val_t *binv = reinterpret_cast<const btn_index_node_val_t *>(e.val);
oid_child = binv->binv_child_oid + m_oid;
} else {
oid_child = *reinterpret_cast<const uint64_t *>(e.val);
}
if (m_omap) {
omap_res_t omr;
m_omap->Lookup(omr, oid_child, m_xid);
// out.st() << "omap: " << omr.oid << " " << omr.xid << " => " << omr.flags << " " << omr.size << " " << omr.paddr << std::endl;
if (omr.flags & OMAP_VAL_DELETED) {
out.st() << "Omap val " << oid_child << " deleted." << std::endl;
continue;
}
}
child = GetNode(oid_child, node, k);
if (child)
DumpTreeInternal(out, child);
else
out.st() << "Child node " << oid_child << " of parent " << oid_parent << " not found!" << std::endl;
}
}
}
std::shared_ptr<BTreeNode> BTree::GetNode(oid_t oid, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index)
{
std::shared_ptr<BTreeNode> node;
bool rc;
// printf("GetNode oid=%" PRIx64 "\n", oid);
#ifdef BTREE_USE_MAP
m_mutex.lock();
auto it = m_nodes.find(oid);
if (it != m_nodes.end())
node = it->second;
m_mutex.unlock();
if (!node)
#endif
{
omap_res_t omr;
omr.oid = oid;
omr.xid = m_xid;
omr.flags = 0;
omr.size = m_treeinfo.bt_fixed.bt_node_size;
omr.paddr = oid;
std::vector<uint8_t> blk;
if (m_omap)
{
rc = m_omap->Lookup(omr, oid, m_xid);
if (g_debug & Dbg_Info) {
std::cout << "omap: oid=" << omr.oid << " xid=" << omr.xid << " flags=" << omr.flags << " size=" << omr.size << " paddr=" << omr.paddr << std::endl;
}
if (!rc)
{
std::cerr << "ERROR: GetNode: omap entry oid " << std::hex << oid << " xid " << m_xid << " not found." << std::endl;
return node;
}
}
blk.resize(m_container.GetBlocksize());
if (m_volume)
{
// TODO: is the crypto_id always equal to the block ID here?
// I think so, the xts id and the block id only differ when the
// volume has been converted from a HFS/FileVault volume, which
// used CoreStorage. After conversions, the block numbers do not
// match anymore, since the CoreStorage data has been removed
// and assigned to the apfs volume. But the metadata is always
// fresh and therefore the ids should match.
if (!m_volume->ReadBlocks(blk.data(), omr.paddr, 1, (omr.flags & OMAP_VAL_ENCRYPTED) ? omr.paddr : 0))
{
std::cerr << "ERROR: GetNode: ReadBlocks failed!" << std::endl;
return node;
}
if (!(omr.flags & OMAP_VAL_NOHEADER)) {
if (!VerifyBlock(blk.data(), blk.size()))
{
std::cerr << "ERROR: GetNode: VerifyBlock failed!" << std::endl;
if (g_debug & Dbg_Errors)
DumpHex(std::cerr, blk.data(), blk.size());
return node;
}
} else {
/*
std::cout << "BTNode @ " << omr.paddr << ":" << std::endl;
DumpHex(std::cout, blk.data(), blk.size());
std::cout << std::endl;
*/
}
}
else
{
if (!m_container.ReadAndVerifyHeaderBlock(blk.data(), omr.paddr))
{
std::cerr << "ERROR: GetNode: ReadAndVerifyHeaderBlock failed!" << std::endl;
return node;
}
}
node = BTreeNode::CreateNode(*this, blk.data(), blk.size(), omr.paddr, parent, parent_index);
#ifdef BTREE_USE_MAP
m_mutex.lock();
if (m_nodes.size() > BTREE_MAP_MAX_NODES)
{
#if 0
m_nodes.clear(); // TODO: Make this somewhat more intelligent ...
#else
// This might be somewhat more intelligent ...
for (it = m_nodes.begin(); it != m_nodes.end();)
{
if (it->second.use_count() == 1)
it = m_nodes.erase(it);
else
++it;
}
#endif
}
m_nodes[oid] = node;
m_mutex.unlock();
#endif
}
return node;
}
uint32_t BTree::Find(const std::shared_ptr<BTreeNode> &node, const void *key, size_t key_size, BTCompareFunc func, void *context)
{
uint32_t k;
uint32_t cnt;
int res_cur = 0;
int res_prev = 0;
BTreeEntry e;
cnt = node->entries_cnt();
if (cnt == 0)
return 0;
// Erst mal linear suchen ...
for (k = 0; k < cnt; k++)
{
node->GetEntry(e, k);
res_cur = func(key, key_size, e.key, e.key_len, context);
if (res_cur > 0 && res_prev <= 0)
break;
res_prev = res_cur;
}
return k;
}
int BTree::FindBin(const std::shared_ptr<BTreeNode>& node, const void* key, size_t key_size, BTCompareFunc func, void *context, FindMode mode)
{
static const char resstr[3] = { '<', '=', '>' };
int beg;
int end;
int mid = -1;
int cnt = node->entries_cnt();
int res;
BTreeEntry e;
int rc;
if (cnt <= 0)
return -1;
beg = 0;
end = cnt - 1;
if (m_debug)
{
std::cout << "FindBin : ";
DumpHex(std::cout, reinterpret_cast<const uint8_t *>(key), key_size, key_size);
}
while (beg <= end)
{
mid = (beg + end) / 2;
node->GetEntry(e, mid);
rc = func(key, key_size, e.key, e.key_len, context);
if (m_debug)
{
std::cout << std::dec << std::setfill(' ');
std::cout << std::setw(2) << beg << " [" << std::setw(2) << mid << "] " << std::setw(2) << end << " : " << resstr[rc + 1] << " : ";
DumpHex(std::cout, reinterpret_cast<const uint8_t *>(e.key), e.key_len, e.key_len);
}
if (rc == 0)
break;
if (rc == -1)
beg = mid + 1;
else if (rc == 1)
end = mid - 1;
}
if (m_debug)
{
std::cout << std::dec << std::setfill(' ');
std::cout << " => " << resstr[rc + 1] << ", " << mid;
}
switch (mode)
{
case FindMode::EQ:
res = (rc == 0) ? mid : -1;
break;
case FindMode::LE:
res = (rc <= 0) ? mid : (mid - 1);
break;
case FindMode::LT:
res = (rc < 0) ? mid : (mid - 1);
break;
case FindMode::GE:
res = (rc >= 0) ? mid : (mid + 1);
break;
case FindMode::GT:
res = (rc > 0) ? mid : (mid + 1);
break;
default:
assert(false);
res = -1;
break;
}
if (res == cnt)
res = -1;
if (m_debug)
std::cout << " => " << res << std::endl;
return res;
}
BTreeIterator::BTreeIterator()
{
m_tree = nullptr;
m_index = 0;
}
BTreeIterator::BTreeIterator(BTree *tree, const std::shared_ptr<BTreeNode> &node, uint32_t index)
{
m_tree = tree;
m_node = node;
m_index = index;
}
BTreeIterator::~BTreeIterator()
{
}
void BTreeIterator::Setup(BTree* tree, const std::shared_ptr<BTreeNode>& node, uint32_t index)
{
m_tree = tree;
m_node = node;
m_index = index;
}
bool BTreeIterator::next()
{
if (!m_node)
return false;
m_index++;
if (m_index < m_node->entries_cnt())
return true;
else {
std::shared_ptr<BTreeNode> node = next_node();
if (node) {
m_index = 0;
m_node = node;
return true;
}
}
return false;
}
void BTreeIterator::reset()
{
m_tree = nullptr;
m_node.reset();
m_index = 0;
}
bool BTreeIterator::GetEntry(BTreeEntry& res) const
{
if (!m_node)
return false;
return m_node->GetEntry(res, m_index);
}
#undef BTITDBG
std::shared_ptr<BTreeNode> BTreeIterator::next_node()
{
std::shared_ptr<BTreeNode> node;
uint32_t pidx;
BTreeEntry e;
oid_t oid;
node = m_node;
#ifdef BTITDBG
std::cout << "======== ******** BTreeIterator::next_node() ******** ========" << std::endl;
std::cout << " Current node: " << node->nodeid() << std::endl;
#endif
do {
pidx = node->parent_index();
node = node->parent();
#ifdef BTITDBG
std::cout << " Navigating up to node " << node->nodeid() << " index " << pidx << std::endl;
#endif
pidx++;
} while (node && pidx >= node->entries_cnt());
if (!node)
return std::shared_ptr<BTreeNode>();
while (node->level() > 0) {
node->GetEntry(e, pidx);
if (node->flags() & BTNODE_HASHED) {
const btn_index_node_val_t *binv = reinterpret_cast<const btn_index_node_val_t *>(e.val);
oid = binv->binv_child_oid + m_tree->m_oid;
} else {
oid = *reinterpret_cast<const uint64_t *>(e.val);
}
#ifdef BTITDBG
std::cout << " Navigating down to node " << oid << std::endl;
#endif
node = m_tree->GetNode(oid, node, pidx);
if (!node)
std::cerr << "Failed to load btree node oid " << oid << std::endl;
pidx = 0;
}
return node;
}
+209
查看文件
@@ -0,0 +1,209 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <vector>
#include <map>
#include <memory>
#include <mutex>
#include "Global.h"
#include "DiskStruct.h"
#include "ApfsNodeMapper.h"
class BTree;
class BTreeNode;
class BTreeIterator;
class BlockDumper;
class ApfsContainer;
class ApfsVolume;
// This enables a rudimentary disk cache ...
#define BTREE_USE_MAP
// TODO: Think about a better solution.
// 8192 will take max. 32 MB of RAM. Higher may be faster, but use more RAM.
#define BTREE_MAP_MAX_NODES 8192
// ekey < skey: -1, ekey > skey: 1, ekey == skey: 0
typedef int(*BTCompareFunc)(const void *skey, size_t skey_len, const void *ekey, size_t ekey_len, void *context);
int CompareStdKey(const void *skey, size_t skey_len, const void *ekey, size_t ekey_len, void *context);
class BTreeEntry
{
friend class BTree;
public:
BTreeEntry();
~BTreeEntry();
BTreeEntry(const BTreeEntry &o) = delete;
BTreeEntry &operator=(const BTreeEntry &o) = delete;
void clear();
const void *key;
const void *val;
size_t key_len;
size_t val_len;
private:
std::shared_ptr<BTreeNode> m_node;
};
class BTreeNode
{
protected:
BTreeNode(BTree &tree, const uint8_t *block, size_t blocksize, paddr_t paddr, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index);
public:
static std::shared_ptr<BTreeNode> CreateNode(BTree &tree, const uint8_t *block, size_t blocksize, paddr_t paddr, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index);
virtual ~BTreeNode();
uint64_t nodeid() const { return m_btn->btn_o.o_oid; }
uint32_t entries_cnt() const { return m_btn->btn_nkeys; }
uint16_t level() const { return m_btn->btn_level; }
uint16_t flags() const { return m_btn->btn_flags; }
paddr_t paddr() const { return m_paddr; }
const std::shared_ptr<BTreeNode> &parent() const { return m_parent; }
uint16_t parent_index() const { return m_parent_index; }
virtual bool GetEntry(BTreeEntry &result, uint32_t index) const = 0;
// virtual uint32_t Find(const void *key, size_t key_size, BTCompareFunc func) const = 0;
const std::vector<uint8_t> &block() const { return m_block; }
protected:
std::vector<uint8_t> m_block;
BTree &m_tree;
uint16_t m_keys_start; // Up
uint16_t m_vals_start; // Dn
const uint32_t m_parent_index;
const std::shared_ptr<BTreeNode> m_parent;
const paddr_t m_paddr;
const btree_node_phys_t *m_btn;
};
class BTreeNodeFix : public BTreeNode
{
public:
BTreeNodeFix(BTree &tree, const uint8_t *block, size_t blocksize, paddr_t paddr, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index);
bool GetEntry(BTreeEntry &result, uint32_t index) const override;
// uint32_t Find(const void *key, size_t key_size, BTCompareFunc func) const override;
private:
const kvoff_t *m_entries;
};
class BTreeNodeVar : public BTreeNode
{
public:
BTreeNodeVar(BTree &tree, const uint8_t *block, size_t blocksize, paddr_t paddr, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index);
bool GetEntry(BTreeEntry &result, uint32_t index) const override;
// uint32_t Find(const void *key, size_t key_size, BTCompareFunc func) const override;
private:
const kvloc_t *m_entries;
};
class BTree
{
enum class FindMode
{
EQ,
LE,
LT,
GE,
GT
};
friend class BTreeIterator;
public:
BTree(ApfsContainer &container, ApfsVolume *vol = nullptr);
~BTree();
bool Init(oid_t oid_root, xid_t xid, ApfsNodeMapper *omap = nullptr);
bool Lookup(BTreeEntry &result, const void *key, size_t key_size, BTCompareFunc func, void *context, bool exact);
bool GetIterator(BTreeIterator &it, const void *key, size_t key_size, BTCompareFunc func, void *context);
bool GetIteratorBegin(BTreeIterator &it);
uint16_t GetKeyLen() const { return m_treeinfo.bt_fixed.bt_key_size; }
uint16_t GetValLen() const { return m_treeinfo.bt_fixed.bt_val_size; }
void dump(BlockDumper &out);
void EnableDebugOutput() { m_debug = true; }
private:
void DumpTreeInternal(BlockDumper &out, const std::shared_ptr<BTreeNode> &node);
uint32_t Find(const std::shared_ptr<BTreeNode> &node, const void *key, size_t key_size, BTCompareFunc func, void *context);
int FindBin(const std::shared_ptr<BTreeNode> &node, const void *key, size_t key_size, BTCompareFunc func, void *context, FindMode mode);
std::shared_ptr<BTreeNode> GetNode(oid_t oid, const std::shared_ptr<BTreeNode> &parent, uint32_t parent_index);
ApfsContainer &m_container;
ApfsVolume *m_volume;
std::shared_ptr<BTreeNode> m_root_node;
ApfsNodeMapper *m_omap;
btree_info_t m_treeinfo;
oid_t m_oid;
xid_t m_xid;
bool m_debug;
#ifdef BTREE_USE_MAP
std::map<uint64_t, std::shared_ptr<BTreeNode>> m_nodes;
std::mutex m_mutex;
#endif
};
class BTreeIterator
{
public:
BTreeIterator();
BTreeIterator(BTree *tree, const std::shared_ptr<BTreeNode> &node, uint32_t index);
~BTreeIterator();
bool next();
void reset();
bool GetEntry(BTreeEntry &res) const;
void Setup(BTree *tree, const std::shared_ptr<BTreeNode> &node, uint32_t index);
private:
BTree *m_tree;
std::shared_ptr<BTreeNode> m_node;
uint32_t m_index;
std::shared_ptr<BTreeNode> next_node();
};
+2203
查看文件
文件差异过大,未显示。 加载差异
+117
查看文件
@@ -0,0 +1,117 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <string>
#include <vector>
#include <ostream>
#include "Global.h"
#include "DiskStruct.h"
struct FlagDesc
{
uint64_t flag;
const char *desc;
};
class BlockDumper
{
public:
BlockDumper(std::ostream &os, size_t blocksize);
~BlockDumper();
void SetTextFlags(uint32_t flags) { m_text_flags = flags; }
size_t GetBlockSize() const { return m_blocksize; }
void SetBlockSize(size_t size) { m_blocksize = size; }
void DumpNode(const uint8_t *block, uint64_t blk_nr);
std::ostream &st() { return m_os; }
private:
typedef void(BlockDumper::*DumpFunc)(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpNodeHeader(const obj_phys_t *blk, uint64_t blk_nr);
void DumpBTNode(DumpFunc func, uint16_t key_size = 0, uint16_t value_size = 0);
void DumpBTHeader(bool dump_offsets = false);
void DumpBTreeInfo();
// void DumpTableHeader(const APFS_TableHeader &tbl);
void DumpBlk_APSB();
void DumpBlk_CAB();
void DumpBlk_CIB();
void DumpBlk_OM();
void DumpBlk_CPM();
void DumpBlk_NXSB();
void DumpBlk_SM();
void DumpBlk_NR();
void DumpBlk_NRL();
void DumpBlk_JSDR();
void DumpBlk_ER();
void DumpBlk_WBC();
void DumpBlk_WBCL();
void DumpBlk_SnapMetaExt();
void DumpBlk_IntegrityMeta();
void DumpBTNode_0();
void DumpBTEntry_APFS_Root(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpBTEntry_OMap(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpBTEntry_APFS_ExtentRef(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpBTEntry_APFS_SnapMeta(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpBTEntry_OMap_Snapshot(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpBTEntry_FreeList(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpBTEntry_GBitmap(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpBTEntry_FusionMT(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpBTEntry_FExtTree(const void* key_ptr, size_t key_len, const void* val_ptr, size_t val_len, bool index);
void DumpBTEntry_Unk(const void *key_ptr, size_t key_len, const void *val_ptr, size_t val_len, bool index);
void DumpBTIndex(const void* val_ptr, uint16_t val_len);
void Dump_XF(const uint8_t *xf_data, size_t xf_size, bool drec);
void DumpBlockHex();
void DumpHex(const uint8_t *data, size_t size, size_t line_size = 16);
static std::string flagstr(uint64_t flag, const FlagDesc *desc);
static std::string enumstr(uint64_t flag, const FlagDesc *desc);
public:
static const char * GetNodeType(uint32_t type, uint32_t subtype);
private:
static std::string tstamp(uint64_t apfs_time);
void dumpm(const char *name, const void *base, const uint8_t &v, bool lf = true);
void dumpm(const char *name, const void *base, const le_uint16_t &v, bool lf = true);
void dumpm(const char *name, const void *base, const le_uint32_t &v, bool lf = true);
void dumpm(const char *name, const void *base, const le_uint64_t &v, bool lf = true);
void dumpm(const char *name, const void *base, const apfs_uuid_t &uuid);
uint32_t m_text_flags; // 00 - Alt, 01 - insensitive, 08 - sensitive
std::ostream &m_os;
const uint8_t *m_block;
size_t m_blocksize;
};
+86
查看文件
@@ -0,0 +1,86 @@
#include <cassert>
#include "ApfsContainer.h"
#include "DiskStruct.h"
#include "BlockDumper.h"
#include "CheckPointMap.h"
CheckPointMap::CheckPointMap(ApfsContainer& container) : m_container(container)
{
m_cpm_oid = 0;
m_blksize = 0;
}
CheckPointMap::~CheckPointMap()
{
}
bool CheckPointMap::Init(oid_t root_oid, uint32_t blk_count)
{
uint32_t n;
const checkpoint_map_phys_t *cpm;
m_blksize = m_container.GetBlocksize();
m_cpm_data.resize(m_blksize * blk_count);
for (n = 0; n < blk_count; n++)
{
if (!m_container.ReadAndVerifyHeaderBlock(m_cpm_data.data() + n * m_blksize, root_oid + n))
{
m_cpm_data.clear();
return false;
}
cpm = reinterpret_cast<const checkpoint_map_phys_t *>(m_cpm_data.data() + m_blksize * n);
assert((cpm->cpm_o.o_type & OBJECT_TYPE_MASK) == OBJECT_TYPE_CHECKPOINT_MAP);
if ((cpm->cpm_o.o_type & OBJECT_TYPE_MASK) != OBJECT_TYPE_CHECKPOINT_MAP)
{
m_cpm_data.clear();
return false;
}
}
m_cpm_oid = root_oid;
return true;
}
bool CheckPointMap::Lookup(omap_res_t & res, oid_t oid, xid_t xid)
{
uint32_t blk_offs;
uint32_t k;
uint32_t cnt;
const checkpoint_map_phys_t * cpm;
(void)xid;
for (blk_offs = 0; blk_offs < m_cpm_data.size(); blk_offs += m_blksize)
{
cpm = reinterpret_cast<const checkpoint_map_phys_t *>(m_cpm_data.data() + blk_offs);
cnt = cpm->cpm_count;
for (k = 0; k < cnt; k++)
{
if (oid == cpm->cpm_map[k].cpm_oid)
{
res.oid = cpm->cpm_map[k].cpm_oid;
res.xid = cpm->cpm_o.o_xid;
res.flags = 0;
res.size = cpm->cpm_map[k].cpm_size;
res.paddr = cpm->cpm_map[k].cpm_paddr;
return true;
}
}
}
return false;
}
void CheckPointMap::dump(BlockDumper& bd)
{
bd.DumpNode(m_cpm_data.data(), m_cpm_oid);
}
+27
查看文件
@@ -0,0 +1,27 @@
#pragma once
#include <vector>
#include "DiskStruct.h"
#include "ApfsNodeMapper.h"
class ApfsContainer;
class BlockDumper;
class CheckPointMap : public ApfsNodeMapper
{
public:
CheckPointMap(ApfsContainer &container);
virtual ~CheckPointMap();
bool Init(oid_t root_oid, uint32_t blk_count);
bool Lookup(omap_res_t & res, oid_t oid, xid_t xid) override;
void dump(BlockDumper &bd);
private:
ApfsContainer &m_container;
std::vector<uint8_t> m_cpm_data;
oid_t m_cpm_oid;
uint32_t m_blksize;
};
+97
查看文件
@@ -0,0 +1,97 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Crc32.h"
Crc32::Crc32(bool reflect, uint32_t poly)
{
unsigned int i;
uint32_t r;
unsigned int b;
m_reflect = reflect;
m_crc = 0;
if (reflect) {
poly = ((poly << 16) & 0xFFFF0000) | ((poly >> 16) & 0x0000FFFF);
poly = ((poly << 8) & 0xFF00FF00) | ((poly >> 8) & 0x00FF00FF);
poly = ((poly << 4) & 0xF0F0F0F0) | ((poly >> 4) & 0x0F0F0F0F);
poly = ((poly << 2) & 0xCCCCCCCC) | ((poly >> 2) & 0x33333333);
poly = ((poly << 1) & 0xAAAAAAAA) | ((poly >> 1) & 0x55555555);
for (i = 0; i < 256; i++) {
r = i;
for (b = 0; b < 8; b++) {
if (r & 1)
r = (r >> 1) ^ poly;
else
r = (r >> 1);
}
m_table[i] = r;
}
}
else {
for (i = 0; i < 256; i++) {
r = (i << 24);
for (b = 0; b < 8; b++) {
if (r & 0x80000000)
r = (r << 1) ^ poly;
else
r = (r << 1);
}
m_table[i] = r;
}
}
}
Crc32::~Crc32()
{
}
void Crc32::Calc(const uint8_t *data, size_t size)
{
size_t i;
for (i = 0; i < size; i++) {
if (m_reflect) {
CalcLE(data[i]);
}
else {
CalcBE(data[i]);
}
}
}
void Crc32::CalcLE(uint8_t b)
{
m_crc = m_table[b ^ (m_crc & 0xFF)] ^ (m_crc >> 8);
}
void Crc32::CalcBE(uint8_t b)
{
m_crc = m_table[b ^ ((m_crc >> 24) & 0xFF)] ^ (m_crc << 8);
}
uint32_t Crc32::GetDataCRC(const uint8_t *data, size_t size, uint32_t initialXor, uint32_t finalXor)
{
m_crc = initialXor;
Calc(data, size);
return m_crc ^ finalXor;
}
+47
查看文件
@@ -0,0 +1,47 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <cstddef>
#include <cstdint>
#include "Global.h"
class Crc32
{
public:
Crc32(bool reflect, uint32_t poly = 0x04C11DB7);
~Crc32();
void SetCRC(uint32_t crc) { m_crc = crc; }
uint32_t GetCRC() const { return m_crc; }
void Calc(const uint8_t *data, size_t size);
uint32_t GetDataCRC(const uint8_t *data, size_t size, uint32_t initialXor, uint32_t finalXor);
private:
void CalcLE(uint8_t b);
void CalcBE(uint8_t b);
uint32_t m_table[256];
uint32_t m_crc;
bool m_reflect;
};
+341
查看文件
@@ -0,0 +1,341 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iomanip>
#include <iostream>
#include <cstring>
#include <cassert>
#include "Decmpfs.h"
#include "Endian.h"
#include "Global.h"
#include "Util.h"
struct RsrcForkHeader
{
be_uint32_t data_offset;
be_uint32_t mgmt_offset;
be_uint32_t data_size;
be_uint32_t mgmt_size;
};
struct CmpfRsrcEntry
{
// 1 64K-Block
le_uint32_t off;
le_uint32_t size;
};
struct CmpfRsrc
{
le_uint32_t entries;
CmpfRsrcEntry entry[32];
};
bool IsDecompAlgoSupported(uint16_t algo)
{
switch (algo)
{
case 3:
case 4:
case 7:
case 8:
case 9:
case 10:
case 11:
case 12:
case 13:
case 14:
return true;
default:
return false;
}
}
bool IsDecompAlgoInRsrc(uint16_t algo)
{
switch (algo)
{
case 4:
case 8:
case 10:
case 12:
case 14:
return true;
default:
return false;
}
}
bool DecompressFile(ApfsDir &dir, uint64_t ino, std::vector<uint8_t> &decompressed, const std::vector<uint8_t> &compressed)
{
if (compressed.size() < sizeof(CompressionHeader))
return false;
const CompressionHeader *hdr = reinterpret_cast<const CompressionHeader *>(compressed.data());
const uint8_t *cdata = compressed.data() + sizeof(CompressionHeader);
size_t csize = compressed.size() - sizeof(CompressionHeader);
size_t decoded_bytes = 0;
#if 1 // Disable to get compressed data
if (g_debug & Dbg_Cmpfs)
{
std::cout << "DecompressFile " << compressed.size() << " => " << hdr->size << ", algo = " << hdr->algo;
switch (hdr->algo)
{
case 3: std::cout << " (Zlib, Attr)"; break;
case 4: std::cout << " (Zlib, Rsrc)"; break;
case 7: std::cout << " (LZVN, Attr)"; break;
case 8: std::cout << " (LZVN, Rsrc)"; break;
case 9: std::cout << " (Uncompressed, Attr)"; break;
case 10: std::cout << " (Uncompressed, Rsrc)"; break;
case 11: std::cout << " (LZFSE, Attr)"; break;
case 12: std::cout << " (LZFSE, Rsrc)"; break;
case 13: std::cout << " (LZBITMAP, Attr)"; break;
case 14: std::cout << " (LZBITMAP, Rsrc)"; break;
default: std::cout << " (Unknown)"; break;
}
std::cout << std::endl;
}
if (!IsDecompAlgoSupported(hdr->algo))
{
if (g_debug & Dbg_Errors) {
std::cout << "Unsupported decompression algorithm." << std::endl;
DumpHex(std::cout, compressed.data(), compressed.size());
}
return false;
}
if (IsDecompAlgoInRsrc(hdr->algo))
{
std::vector<uint8_t> rsrc;
size_t k;
bool rc = dir.GetAttribute(rsrc, ino, "com.apple.ResourceFork");
if (!rc)
{
if (g_debug & Dbg_Errors)
std::cout << "Decmpfs: Could not find resource fork " << ino << std::endl;
decompressed.clear();
return false;
}
if (hdr->algo == 4) // Zlib, rsrc
{
RsrcForkHeader rsrc_hdr;
memcpy(&rsrc_hdr, rsrc.data(), sizeof(rsrc_hdr));
if (rsrc_hdr.data_offset > rsrc.size())
{
if (g_debug & Dbg_Errors)
std::cout << "Decmpfs: Invalid data offset in rsrc header." << std::endl;
return false;
}
const uint8_t *cmpf_rsrc_base = rsrc.data() + rsrc_hdr.data_offset + sizeof(uint32_t);
const CmpfRsrc *cmpf_rsrc = reinterpret_cast<const CmpfRsrc *>(cmpf_rsrc_base);
decompressed.resize((hdr->size + 0xFFFF) & 0xFFFF0000);
for (k = 0; k < cmpf_rsrc->entries; k++)
{
size_t src_offset = cmpf_rsrc->entry[k].off;
const uint8_t *src = cmpf_rsrc_base + src_offset;
size_t src_len = cmpf_rsrc->entry[k].size;
uint8_t *dst = decompressed.data() + 0x10000 * k;
size_t expected_len = hdr->size - (0x10000 * k);
if (expected_len > 0x10000)
expected_len = 0x10000;
if (src_len > 0x10001)
{
if (g_debug & Dbg_Errors)
std::cout << "Decmpfs: In rsrc, src_len too big (" << src_len << ")" << std::endl;
return false;
}
if (src[0] == 0x78)
{
decoded_bytes = DecompressZLib(dst, 0x10000, src, src_len);
}
else if ((src[0] & 0x0F) == 0x0F)
{
memcpy(dst, src + 1, src_len - 1);
decoded_bytes = src_len - 1;
}
else
{
if (g_debug & Dbg_Errors)
std::cout << "Decmpfs: Something wrong with zlib data." << std::endl;
decompressed.clear();
return false;
}
if (expected_len != decoded_bytes)
{
if (g_debug & Dbg_Errors)
std::cout << "Decmpfs: Expected len != decompressed len: " << expected_len << " != " << decoded_bytes << std::endl;
return false;
}
}
}
else
{
const uint32_t *off_list = reinterpret_cast<const uint32_t *>(rsrc.data());
decompressed.resize((hdr->size + 0xFFFF) & 0xFFFF0000);
for (k = 0; (k << 16) < decompressed.size(); k++) {
size_t expected_len = hdr->size - (0x10000 * k);
if (expected_len > 0x10000)
expected_len = 0x10000;
const uint8_t *src = rsrc.data() + off_list[k];
size_t src_len = off_list[k + 1] - off_list[k];
if (src_len > 0x10001) {
if (g_debug & Dbg_Errors)
std::cout << "Decmpfs: In rsrc, src_len too big (" << src_len << ")" << std::endl;
return false;
}
switch (hdr->algo) {
case 8:
if (src[0] == 0x06) {
memcpy(decompressed.data() + (k << 16), src + 1, src_len - 1);
decoded_bytes = src_len - 1;
}
else
decoded_bytes = DecompressLZVN(decompressed.data() + (k << 16), expected_len, src, src_len);
break;
case 10:
// Assuming ...
memcpy(decompressed.data() + (k << 16), src + 1, src_len - 1);
decoded_bytes = src_len - 1;
break;
case 12:
// Assuming ...
decoded_bytes = DecompressLZFSE(decompressed.data() + (k << 16), expected_len, src, src_len);
// TODO is there also an uncompressed variant?
break;
case 14:
if (src[0] == 0xFF) {
memcpy(decompressed.data() + (k << 16), src + 1, src_len - 1);
decoded_bytes = src_len - 1;
}
else
decoded_bytes = DecompressLZBITMAP(decompressed.data() + (k << 16), expected_len, src, src_len);
break;
default:
decoded_bytes = 0;
break;
}
if (decoded_bytes != expected_len) {
if (g_debug & Dbg_Errors)
std::cout << "Decmpfs: Expected length != decompressed length: " << expected_len << " != " << decoded_bytes << " [k = " << k << "]" << std::endl;
return false;
}
}
}
decompressed.resize(hdr->size);
}
else
{
decompressed.resize(hdr->size);
switch (hdr->algo) {
case 3:
if (cdata[0] == 0x78)
decoded_bytes = DecompressZLib(decompressed.data(), decompressed.size(), cdata, csize);
else if (cdata[0] == 0xFF) {
assert(hdr->size == csize - 1);
decompressed.assign(cdata + 1, cdata + csize);
decoded_bytes = decompressed.size();
}
else
return false;
break;
case 7:
if (cdata[0] == 0x06) {
assert(hdr->size == csize - 1);
decompressed.assign(cdata + 1, cdata + csize);
decoded_bytes = decompressed.size();
}
else
decoded_bytes = DecompressLZVN(decompressed.data(), decompressed.size(), cdata, csize);
break;
case 9:
assert(cdata[0] == 0xCC);
assert(hdr->size == csize - 1);
decompressed.assign(cdata + 1, cdata + csize);
decoded_bytes = decompressed.size();
break;
case 11:
// TODO uncompressed variant?
decoded_bytes = DecompressLZFSE(decompressed.data(), decompressed.size(), cdata, csize);
break;
case 13:
if (cdata[0] == 0xFF) {
assert(hdr->size == csize - 1);
decompressed.assign(cdata + 1, cdata + csize);
decoded_bytes = decompressed.size();
}
else
decoded_bytes = DecompressLZBITMAP(decompressed.data(), decompressed.size(), cdata, csize);
break;
default:
decoded_bytes = 0;
break;
}
if (decoded_bytes != hdr->size)
{
if (g_debug & Dbg_Errors)
std::cout << "Decmpfs: In attr, expected len != decoded len: " << hdr->size << " != " << decoded_bytes << std::endl;
return false;
}
}
#else
if (IsDecompAlgoInRsrc(hdr->algo))
{
dir.GetAttribute(decompressed, ino, "com.apple.ResourceFork");
}
else
{
decompressed = compressed;
}
#endif
return true;
}
+36
查看文件
@@ -0,0 +1,36 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <vector>
#include "ApfsDir.h"
struct CompressionHeader
{
le_uint32_t signature;
le_uint32_t algo;
le_uint64_t size;
};
bool IsDecompAlgoSupported(uint16_t algo);
bool IsDecompAlgoInRsrc(uint16_t algo);
bool DecompressFile(ApfsDir &dir, uint64_t ino, std::vector<uint8_t> &decompressed, const std::vector<uint8_t> &compressed);
+111
查看文件
@@ -0,0 +1,111 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <cstring>
#include "Device.h"
#include "DeviceWinFile.h"
#include "DeviceWinPhys.h"
#include "DeviceLinux.h"
#include "DeviceMac.h"
#include "DeviceDMG.h"
#include "DeviceSparseImage.h"
#include "DeviceVDI.h"
Device::Device()
{
m_sector_size = 0x200;
}
Device::~Device()
{
}
Device * Device::OpenDevice(const char * name)
{
Device *dev = nullptr;
bool rc;
const char *ext;
#ifdef _WIN32
if (!strncmp(name, "\\\\.\\PhysicalDrive", 17))
{
dev = new DeviceWinPhys();
rc = dev->Open(name);
if (rc)
return dev;
else
{
dev->Close();
delete dev;
dev = nullptr;
}
}
#endif
ext = strrchr(name, '.');
if (ext)
{
if (!strcmp(ext, ".dmg"))
{
DeviceDMG *dmg;
dmg = new DeviceDMG();
rc = dmg->Open(name);
if (rc)
return dmg;
dmg->Close();
delete dmg;
}
if (!strcmp(ext, ".sparseimage"))
{
DeviceSparseImage *sprs;
sprs = new DeviceSparseImage();
rc = sprs->Open(name);
if (rc)
return sprs;
sprs->Close();
delete sprs;
}
}
if (!dev)
{
#ifdef _WIN32
dev = new DeviceWinFile();
#endif
#ifdef __linux__
dev = new DeviceLinux();
#endif
#ifdef __APPLE__
dev = new DeviceMac();
#endif
rc = dev->Open(name);
if (!rc)
{
dev->Close();
delete dev;
dev = nullptr;
}
}
return dev;
}
+45
查看文件
@@ -0,0 +1,45 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <cstdint>
class Device
{
protected:
Device();
public:
virtual ~Device();
virtual bool Open(const char *name) = 0;
virtual void Close() = 0;
virtual bool Read(void *data, uint64_t offs, uint64_t len) = 0;
virtual uint64_t GetSize() const = 0;
unsigned int GetSectorSize() const { return m_sector_size; }
void SetSectorSize(unsigned int size) { m_sector_size = size; }
static Device *OpenDevice(const char *name);
private:
unsigned int m_sector_size;
};
+527
查看文件
@@ -0,0 +1,527 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <cstring>
#include <vector>
#include <iostream>
#include <iomanip>
#include "Endian.h"
#include "Util.h"
#include "PList.h"
#include "DeviceDMG.h"
#pragma pack(push, 1)
struct KolyHeader
{
char signature[4];
be_uint32_t version;
be_uint32_t headersize;
be_uint32_t flags;
be_uint64_t running_data_fork_offset;
be_uint64_t data_fork_offset;
be_uint64_t data_fork_length;
be_uint64_t rsrc_fork_offset;
be_uint64_t rsrc_fork_length;
be_uint32_t segment_number;
be_uint32_t segment_count;
uint8_t segment_id[0x10];
// 0x50
be_uint32_t data_fork_checksum_type;
be_uint32_t data_fork_unknown;
be_uint32_t data_fork_checksum_data;
uint8_t unk_05C[0x7C];
// 0xD8
be_uint64_t xml_offset;
be_uint64_t xml_length;
// ???
// uint64_t codesign_offset;
// uint64_t codesign_length;
uint8_t unk_0E8[0x78];
// 0x160
be_uint32_t master_checksum_type;
be_uint32_t master_checksum_unknown;
be_uint32_t master_checksum_data;
uint8_t unk_16C[0x7C];
// 0x1E8
be_uint32_t image_variant;
be_uint64_t sector_count;
// 0x1F4
uint8_t unk_1F4[12];
};
static_assert(sizeof(KolyHeader) == 0x200, "Wrong Koly Header Size");
struct MishHeader
{
char signature[4];
be_uint32_t unk;
be_uint64_t sector_start;
be_uint64_t sector_count;
be_uint64_t dmg_offset;
be_uint32_t unk_20;
be_uint32_t part_id;
be_uint8_t unk_28[0x18];
be_uint32_t checksum_type;
be_uint32_t checksum_unk;
be_uint32_t checksum_data;
be_uint8_t unk_4C[0x7C];
be_uint32_t entry_count;
};
static_assert(sizeof(MishHeader) == 0xCC, "Wrong Mish Header Size");
struct MishEntry
{
be_uint32_t method;
be_uint32_t comment;
be_uint64_t sector_start;
be_uint64_t sector_count;
be_uint64_t dmg_offset;
be_uint64_t dmg_length;
};
static_assert(sizeof(MishEntry) == 0x28, "Wrong Mish Entry Size");
#pragma pack(pop)
DeviceDMG::DmgSection::DmgSection()
{
method = 0;
comment = 0;
disk_offset = 0;
disk_length = 0;
dmg_offset = 0;
dmg_length = 0;
cache = nullptr;
}
DeviceDMG::DmgSection::~DmgSection()
{
delete[] cache;
}
DeviceDMG::DeviceDMG() : m_crc(true)
{
m_size = 0;
m_offset = 0;
m_is_raw = false;
#ifdef DMG_CACHE
m_cache_base = 0;
m_cache_size = 0;
m_cache_data = 0;
#endif
}
DeviceDMG::~DeviceDMG()
{
Close();
#ifdef DMG_CACHE
delete[] m_cache_data;
#endif
}
bool DeviceDMG::Open(const char * name)
{
Close();
if (!m_img.Open(name))
return false;
if (!m_img.CheckSetupEncryption())
{
m_img.Close();
return false;
}
KolyHeader koly;
m_img.Read(m_img.GetContentSize() - 0x200, &koly, sizeof(koly));
if (memcmp(koly.signature, "koly", 4))
{
m_is_raw = true;
m_size = m_img.GetContentSize();
return true;
}
#ifdef DMG_DEBUG
m_dbg.open("mish_ev1.txt");
m_dbg.flags(m_dbg.hex | m_dbg.uppercase);
m_dbg.fill('0');
m_dbg << "Signature : " << std::setw(8) << koly.signature << std::endl;
m_dbg << "Version : " << std::setw(8) << koly.version << std::endl;
m_dbg << "Header size : " << std::setw(8) << koly.headersize << std::endl;
m_dbg << "Flags : " << std::setw(8) << koly.flags << std::endl;
m_dbg << "Running DF Off : " << std::setw(16) << koly.running_data_fork_offset << std::endl;
m_dbg << "Data Fork Offs : " << std::setw(16) << koly.data_fork_offset << std::endl;
m_dbg << "Data Fork Len : " << std::setw(16) << koly.data_fork_length << std::endl;
m_dbg << "Rsrc Fork Off : " << std::setw(16) << koly.rsrc_fork_offset << std::endl;
m_dbg << "Rsrc Fork Len : " << std::setw(16) << koly.rsrc_fork_length << std::endl;
m_dbg << "Segment Number : " << std::setw(8) << koly.segment_number << std::endl;
m_dbg << "Segment Count : " << std::setw(8) << koly.segment_count << std::endl;
m_dbg << "..." << std::endl;
m_dbg << "XML Offset : " << std::setw(16) << koly.xml_offset << std::endl;
m_dbg << "XML Length : " << std::setw(16) << koly.xml_length << std::endl;
m_dbg << "..." << std::endl;
DumpHex(m_dbg, reinterpret_cast<const uint8_t *>(&koly), sizeof(koly), 0x20);
m_dbg << std::endl << std::endl;
#endif
m_size = koly.sector_count * 0x200;
m_offset = koly.data_fork_offset;
if (koly.xml_offset != 0)
{
if (g_debug & Dbg_Info)
printf("Loading DMG using XML plist.\n");
if (!ProcessHeaderXML(koly.xml_offset, koly.xml_length))
{
fprintf(stderr, "Error parsing propertiy list.\n");
m_img.Close();
m_img.Reset();
m_size = 0;
return false;
}
}
else if (koly.rsrc_fork_offset != 0)
{
if (g_debug & Dbg_Info)
printf("Loading DMG using RSRC fork.\n");
if (!ProcessHeaderRsrc(koly.rsrc_fork_offset, koly.rsrc_fork_length))
{
fprintf(stderr, "Error processing resource fork.\n");
m_img.Close();
m_img.Reset();
m_size = 0;
return false;
}
}
else
{
m_img.Close();
m_img.Reset();
m_size = 0;
return false;
}
#ifdef DMG_DEBUG
m_dbg.close();
#endif
return true;
}
void DeviceDMG::Close()
{
m_img.Close();
m_size = 0;
m_sections.clear();
m_is_raw = false;
}
bool DeviceDMG::Read(void * data, uint64_t offs, uint64_t len)
{
if (m_is_raw)
{
m_img.Read(offs + m_offset, data, len);
return true;
// TODO: Error handling ...
}
// Binary search start sector in m_sections
// Get data if necessary
// Decompress: cache data
size_t entry_idx = m_sections.size();
size_t rd_offs;
size_t rd_size;
char *bdata = reinterpret_cast<char *>(data);
bool compressed = false;
ptrdiff_t beg = 0;
ptrdiff_t end = m_sections.size() - 1;
ptrdiff_t mid;
while (beg <= end)
{
mid = (beg + end) / 2;
if (offs >= m_sections[mid].disk_offset && offs < (m_sections[mid].disk_offset + m_sections[mid].disk_length))
{
entry_idx = mid;
break;
}
else if (offs < m_sections[mid].disk_offset)
end = mid - 1;
else
beg = mid + 1;
}
if (entry_idx == m_sections.size())
return false;
while (len > 0)
{
if (entry_idx >= m_sections.size())
return false;
DmgSection &sect = m_sections[entry_idx];
rd_offs = offs - sect.disk_offset;
rd_size = len;
if (rd_offs + rd_size > sect.disk_length)
rd_size = sect.disk_length - rd_offs;
compressed = false;
switch (sect.method)
{
case 1: // raw
m_img.Read(rd_offs + sect.dmg_offset + m_offset, bdata, rd_size);
break;
case 0: // unsure ...
case 2: // ignore
memset(bdata, 0, rd_size);
break;
case 0x80000004: // adc
case 0x80000005: // zlib
case 0x80000006: // bzip2
case 0x80000007: // lzfse
compressed = true;
break;
default:
std::cerr << "DMG: unknown compression method " << sect.method << std::endl;
return false;
}
if (compressed)
{
#ifdef DMG_CACHE
if (m_cache_data == 0 || m_cache_base != sect.disk_offset || m_cache_size != sect.disk_length)
{
delete[] m_cache_data;
m_cache_data = 0;
uint8_t *compr_buf = new uint8_t[sect.dmg_length];
m_cache_data = new uint8_t[sect.disk_length];
m_cache_base = sect.disk_offset;
m_cache_size = sect.disk_length;
m_img.Read(sect.dmg_offset + m_offset, compr_buf, sect.dmg_length);
switch (sect.method)
{
case 0x80000004:
DecompressADC(m_cache_data, sect.disk_length, compr_buf, sect.dmg_length);
break;
case 0x80000005:
DecompressZLib(m_cache_data, sect.disk_length, compr_buf, sect.dmg_length);
break;
case 0x80000006:
DecompressBZ2(m_cache_data, sect.disk_length, compr_buf, sect.dmg_length);
break;
case 0x80000007:
DecompressLZFSE(m_cache_data, sect.disk_length, compr_buf, sect.dmg_length);
break;
default:
std::cerr << "DMG: invalid compression method " << sect.method << std::endl;
return false;
break;
}
delete[] compr_buf;
}
memcpy(bdata, m_cache_data + rd_offs, rd_size);
#else
if (sect.cache == nullptr)
{
uint8_t *compr_buf = new uint8_t[sect.dmg_length];
sect.cache = new uint8_t[sect.disk_length];
m_img.Read(sect.dmg_offset + m_offset, compr_buf, sect.dmg_length);
switch (sect.method)
{
case 0x80000004:
DecompressADC(sect.cache, sect.disk_length, compr_buf, sect.dmg_length);
break;
case 0x80000005:
DecompressZLib(sect.cache, sect.disk_length, compr_buf, sect.dmg_length);
break;
case 0x80000006:
DecompressBZ2(sect.cache, sect.disk_length, compr_buf, sect.dmg_length);
break;
case 0x80000007:
DecompressLZFSE(sect.cache, sect.disk_length, compr_buf, sect.dmg_length);
break;
default:
std::cerr << "DMG: invalid compression method " << sect.method << std::endl;
return false;
break;
}
delete[] compr_buf;
memcpy(bdata, sect.cache + rd_offs, rd_size);
#if 1 // TODO: figure out some intelligent cache mechanism ...
delete[] sect.cache;
sect.cache = nullptr;
#endif
}
#endif
}
bdata += rd_size;
offs += rd_size;
len -= rd_size;
entry_idx++;
}
return true;
}
uint64_t DeviceDMG::GetSize() const
{
return m_size;
}
bool DeviceDMG::ProcessHeaderXML(uint64_t off, uint64_t size)
{
std::vector<char> xmldata;
xmldata.resize(size, 0);
m_img.Read(off, xmldata.data(), size);
PListXmlParser parser(xmldata.data(), xmldata.size());
const PLDict *plist = parser.Parse()->toDict();
if (!plist)
return false;
const PLDict *rsrc_fork = plist->get("resource-fork")->toDict();
if (!rsrc_fork)
return false;
const PLArray *blkx = rsrc_fork->get("blkx")->toArray();
if (!blkx)
return false;
const PLDict *entry;
const PLData *mish;
size_t k;
for (k = 0; k < blkx->size(); k++)
{
entry = blkx->get(k)->toDict();
if (!entry)
return false;
mish = entry->get("Data")->toData();
if (!mish)
return false;
ProcessMish(mish->data(), mish->size());
}
return true;
}
bool DeviceDMG::ProcessHeaderRsrc(uint64_t off, uint64_t size)
{
return false;
}
void DeviceDMG::ProcessMish(const uint8_t * data, size_t size)
{
(void)size;
const MishHeader *mish = reinterpret_cast<const MishHeader *>(data);
const MishEntry *entry = reinterpret_cast<const MishEntry *>(data + 0xCC);
DmgSection section;
uint64_t partition_start;
uint32_t cnt;
uint32_t k;
if (memcmp(mish->signature, "mish", 4))
return;
#ifdef DMG_DEBUG
DumpHex(m_dbg, data, 0xCC, 0x20);
m_dbg << std::setw(8) << mish->signature << ' ';
m_dbg << std::setw(8) << mish->unk << ' ';
m_dbg << std::setw(16) << mish->sector_start << ' ';
m_dbg << std::setw(16) << mish->sector_count << ' ';
m_dbg << std::setw(16) << mish->dmg_offset << ' ';
m_dbg << std::setw(8) << mish->unk_20 << ' ';
m_dbg << std::setw(8) << mish->part_id << " - ";
m_dbg << std::setw(8) << mish->checksum_data << std::endl;
#endif
partition_start = mish->sector_start;
cnt = mish->entry_count;
for (k = 0; k < cnt; k++)
{
#ifdef DMG_DEBUG
m_dbg << std::setw(8) << entry[k].method << ' ';
m_dbg << std::setw(8) << entry[k].comment << ' ';
m_dbg << std::setw(16) << entry[k].sector_start << ' ';
m_dbg << std::setw(16) << entry[k].sector_count << ' ';
m_dbg << std::setw(16) << entry[k].dmg_offset << ' ';
m_dbg << std::setw(16) << entry[k].dmg_length << std::endl;
#endif
section.method = entry[k].method;
section.comment = entry[k].comment;
section.disk_offset = (entry[k].sector_start + partition_start) * 0x200;
section.disk_length = entry[k].sector_count * 0x200;
section.dmg_offset = entry[k].dmg_offset + mish->dmg_offset;
section.dmg_length = entry[k].dmg_length;
section.cache = nullptr;
if (section.method != 0xFFFFFFFF && section.method != 0x7FFFFFFE)
m_sections.push_back(section);
}
#ifdef DMG_DEBUG
m_dbg << std::endl;
#endif
}
+85
查看文件
@@ -0,0 +1,85 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <cstdint>
#include <fstream>
#include <string>
#include <vector>
#include "Device.h"
#include "DiskImageFile.h"
#include "Crc32.h"
#undef DMG_DEBUG
#define DMG_CACHE
class DeviceDMG : public Device
{
struct DmgSection
{
DmgSection();
~DmgSection();
uint32_t method;
uint32_t comment;
uint64_t disk_offset;
uint64_t disk_length;
uint64_t dmg_offset;
uint64_t dmg_length;
uint8_t *cache;
};
public:
DeviceDMG();
~DeviceDMG();
bool Open(const char *name) override;
void Close() override;
bool Read(void *data, uint64_t offs, uint64_t len) override;
uint64_t GetSize() const override;
private:
bool ProcessHeaderXML(uint64_t off, uint64_t size);
bool ProcessHeaderRsrc(uint64_t off, uint64_t size);
void ProcessMish(const uint8_t *data, size_t size);
DiskImageFile m_img;
uint64_t m_size;
uint64_t m_offset;
bool m_is_raw;
Crc32 m_crc;
std::vector<DmgSection> m_sections;
#ifdef DMG_DEBUG
std::ofstream m_dbg;
#endif
#ifdef DMG_CACHE
uint64_t m_cache_base;
uint64_t m_cache_size;
uint8_t *m_cache_data;
#endif
};
+94
查看文件
@@ -0,0 +1,94 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef __linux__
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/fcntl.h>
#include <linux/fs.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <iostream>
#include "DeviceLinux.h"
#include "Global.h"
DeviceLinux::DeviceLinux()
{
m_device = -1;
m_size = 0;
}
DeviceLinux::~DeviceLinux()
{
Close();
}
bool DeviceLinux::Open(const char* name)
{
m_device = open(name, O_RDONLY | O_LARGEFILE);
if (m_device == -1)
{
std::cout << "Opening device " << name << " failed with error " << strerror(errno) << std::endl;
return false;
}
struct stat64 st;
fstat64(m_device, &st);
if (S_ISREG(st.st_mode))
{
m_size = st.st_size;
}
else if (S_ISBLK(st.st_mode))
{
// Hmmm ...
ioctl(m_device, BLKGETSIZE64, &m_size);
}
if (g_debug & Dbg_Info)
std::cout << "Device " << name << " opened. Size is " << m_size << std::endl;
return m_device != -1;
}
void DeviceLinux::Close()
{
if (m_device != -1)
close(m_device);
m_device = -1;
m_size = 0;
}
bool DeviceLinux::Read(void* data, uint64_t offs, uint64_t len)
{
size_t nread;
nread = pread64(m_device, data, len, offs);
// TODO: Better error handling ...
return nread == len;
}
#endif
+45
查看文件
@@ -0,0 +1,45 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#ifdef __linux__
#include <cstddef>
#include "Device.h"
class DeviceLinux : public Device
{
public:
DeviceLinux();
~DeviceLinux();
bool Open(const char *name) override;
void Close() override;
bool Read(void *data, uint64_t offs, uint64_t len) override;
uint64_t GetSize() const override { return m_size; }
private:
int m_device;
uint64_t m_size;
};
#endif
+109
查看文件
@@ -0,0 +1,109 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef __APPLE__
#include <unistd.h>
#include <sys/disk.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/fcntl.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <iostream>
#include "DeviceMac.h"
#include "Global.h"
DeviceMac::DeviceMac()
{
m_device = -1;
m_size = 0;
}
DeviceMac::~DeviceMac()
{
Close();
}
bool DeviceMac::Open(const char* name)
{
m_device = open(name, O_RDONLY);
if (m_device == -1)
{
std::cout << "Opening device " << name << " failed with error " << strerror(errno) << std::endl;
return false;
}
struct stat st;
fstat(m_device, &st);
std::cout << "st_mode = " << st.st_mode << std::endl;
if (S_ISREG(st.st_mode))
{
m_size = st.st_size;
}
else if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))
{
uint64_t sector_count = 0;
uint32_t sector_size = 0;
ioctl(m_device, DKIOCGETBLOCKCOUNT, &sector_count);
ioctl(m_device, DKIOCGETBLOCKSIZE, &sector_size);
m_size = sector_size * sector_count;
std::cout << "Sector count = " << sector_count << std::endl;
std::cout << "Sector size = " << sector_size << std::endl;
}
else
{
std::cout << "File mode unknown!" << std::endl;
}
if (g_debug & Dbg_Info)
std::cout << "Device " << name << " opened. Size is " << m_size << std::endl;
return m_device != -1;
}
void DeviceMac::Close()
{
if (m_device != -1)
close(m_device);
m_device = -1;
m_size = 0;
}
bool DeviceMac::Read(void* data, uint64_t offs, uint64_t len)
{
size_t nread;
nread = pread(m_device, data, len, offs);
// TODO: Better error handling ...
return nread == len;
}
#endif
+44
查看文件
@@ -0,0 +1,44 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#ifdef __APPLE__
#include "Device.h"
class DeviceMac : public Device
{
public:
DeviceMac();
~DeviceMac();
bool Open(const char *name) override;
void Close() override;
bool Read(void *data, uint64_t offs, uint64_t len) override;
uint64_t GetSize() const override { return m_size; }
private:
int m_device;
uint64_t m_size;
};
#endif
+188
查看文件
@@ -0,0 +1,188 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include <cstring>
#include "Endian.h"
#include "DeviceSparseImage.h"
#ifdef _MSC_VER
#pragma pack(push, 4)
#define __attribute__(x)
#endif
struct HeaderNode
{
be_uint32_t signature;
be_uint32_t version;
be_uint32_t sectors_per_band;
be_uint32_t flags;
be_uint32_t total_sectors_low;
be_uint64_t next_node_offset;
be_uint64_t total_sectors;
be_uint32_t padding[7];
be_uint32_t band_id[0x3F0];
} __attribute__((packed, aligned(4)));
struct IndexNode
{
be_uint32_t magic;
be_uint32_t index_node_number;
be_uint32_t flags;
be_uint64_t next_node_offset;
be_uint32_t padding[9];
be_uint32_t band_id[0x3F2];
} __attribute__((packed, aligned(4)));
#ifdef _MSC_VER
#pragma pack(pop)
#undef __attribute__
#endif
constexpr int SECTOR_SIZE = 0x200;
constexpr size_t NODE_SIZE = 0x1000;
constexpr size_t BAND_SIZE = 0x100000;
constexpr uint32_t SPRS_SIGNATURE = 0x73707273;
DeviceSparseImage::DeviceSparseImage()
{
m_band_size = 0;
m_size = 0;
}
DeviceSparseImage::~DeviceSparseImage()
{
m_img.Close();
m_img.Reset();
}
bool DeviceSparseImage::Open(const char * name)
{
if (!m_img.Open(name))
return false;
if (!m_img.CheckSetupEncryption())
{
m_img.Close();
m_img.Reset();
return false;
}
HeaderNode hdr;
IndexNode idx;
uint32_t off;
uint64_t base;
uint64_t next;
size_t k;
m_img.Read(0, &hdr, sizeof(hdr));
if (hdr.signature != SPRS_SIGNATURE)
{
m_img.Close();
m_img.Reset();
return false;
}
m_size = hdr.total_sectors * SECTOR_SIZE;
m_band_size = hdr.sectors_per_band * SECTOR_SIZE;
m_band_offset.resize((m_size + m_band_size - 1) / m_band_size, 0);
base = 0x1000;
for (k = 0; k < 0x3F0; k++)
{
off = hdr.band_id[k];
if (off)
m_band_offset[off - 1] = base + m_band_size * k;
}
next = hdr.next_node_offset;
base = next + NODE_SIZE;
while (next)
{
m_img.Read(next, &idx, sizeof(idx));
if (hdr.signature != SPRS_SIGNATURE)
{
m_img.Close();
m_img.Reset();
return false;
}
for (k = 0; k < 0x3F2; k++)
{
off = idx.band_id[k];
if (off)
m_band_offset[off - 1] = base + m_band_size * k;
}
next = idx.next_node_offset;
base = next + NODE_SIZE;
}
return true;
}
void DeviceSparseImage::Close()
{
m_img.Close();
m_img.Reset();
}
bool DeviceSparseImage::Read(void * data, uint64_t offs, uint64_t len)
{
uint32_t chunk;
uint32_t chunk_offs;
uint64_t chunk_base;
size_t read_size;
uint8_t *bdata = reinterpret_cast<uint8_t *>(data);
while (len > 0)
{
chunk = offs >> 20; // TODO
chunk_offs = offs & (m_band_size - 1);
read_size = len;
if ((chunk_offs + read_size) > m_band_size)
read_size = m_band_size - chunk_offs;
chunk_base = m_band_offset[chunk];
if (chunk_base)
m_img.Read(chunk_base + chunk_offs, bdata, read_size);
else
memset(bdata, 0, read_size);
len -= read_size;
offs += read_size;
bdata += read_size;
}
return true;
}
uint64_t DeviceSparseImage::GetSize() const
{
return m_size;
}
+26
查看文件
@@ -0,0 +1,26 @@
#pragma once
#include <vector>
#include "Device.h"
#include "DiskImageFile.h"
class DeviceSparseImage : public Device
{
public:
DeviceSparseImage();
~DeviceSparseImage();
bool Open(const char *name) override;
void Close() override;
bool Read(void *data, uint64_t offs, uint64_t len) override;
uint64_t GetSize() const override;
private:
std::vector<uint64_t> m_band_offset;
uint64_t m_size;
uint64_t m_band_size;
DiskImageFile m_img;
};
+154
查看文件
@@ -0,0 +1,154 @@
#include "DeviceVDI.h"
#pragma pack(1)
struct VdiDiskGeometry
{
le_uint32_t cylinders;
le_uint32_t heads;
le_uint32_t sectors;
le_uint32_t sector_size;
};
struct VdiPreHeader
{
char file_info[0x40];
le_uint32_t signature; // 0xBEDA107F
le_uint32_t version; // 0x00010001
};
struct VdiHeader1Plus
{
le_uint32_t struct_size;
le_uint32_t image_type;
le_uint32_t flags;
char comment[0x100];
le_uint32_t off_blocks;
le_uint32_t off_data;
VdiDiskGeometry legacy_geometry;
le_uint32_t dummy;
le_uint64_t disk_size;
le_uint32_t block_size;
le_uint32_t block_extra;
le_uint32_t blocks_total;
le_uint32_t blocks_allocated;
char uuid_create[0x10];
char uuid_modify[0x10];
char uuid_linkage[0x10];
char uuid_parent_modify[0x10];
VdiDiskGeometry lchs_geometry;
};
#pragma pack()
DeviceVDI::DeviceVDI()
{
m_disk_size = 0;
m_block_size = 0;
m_block_count = 0;
m_data_offset = 0;
m_vdi = nullptr;
}
DeviceVDI::~DeviceVDI()
{
}
bool DeviceVDI::Open(const char * name)
{
VdiPreHeader phdr;
VdiHeader1Plus hdr;
Close();
fopen_s(&m_vdi, name, "rb");
if (m_vdi)
{
if (fread(&phdr, sizeof(phdr), 1, m_vdi) == 1)
{
if (fread(&hdr, sizeof(hdr), 1, m_vdi) == 1)
{
if (phdr.signature == 0xBEDA107F && phdr.version == 0x00010001)
{
m_disk_size = hdr.disk_size;
m_block_size = hdr.block_size;
m_block_count = hdr.blocks_total;
m_data_offset = hdr.off_data;
m_block_map.resize(m_block_count, 0xFF);
fseek(m_vdi, hdr.off_blocks, SEEK_SET);
fread(m_block_map.data(), sizeof(uint32_t), m_block_count, m_vdi);
fseek(m_vdi, 0, SEEK_END);
return true;
}
}
}
}
Close();
return false;
}
void DeviceVDI::Close()
{
if (m_vdi)
{
fclose(m_vdi);
m_vdi = nullptr;
}
}
bool DeviceVDI::Read(void * data, uint64_t offs, uint64_t len)
{
uint64_t block_nr;
uint64_t block_offs;
uint64_t vdi_offs;
uint32_t map_entry;
uint64_t read_size;
uint8_t *pdata = reinterpret_cast<uint8_t *>(data);
if (!m_vdi)
return false;
while (len > 0)
{
block_nr = offs >> 20;
block_offs = offs & 0xFFFFF;
map_entry = m_block_map[block_nr];
read_size = m_block_size - block_offs;
if (read_size > len)
read_size = len;
if (map_entry == 0xFFFFFFFF)
{
memset(pdata, 0, read_size);
}
else
{
fseek(m_vdi, map_entry * m_block_size + m_data_offset + block_offs, SEEK_SET);
fread(pdata, 1, read_size, m_vdi);
}
pdata += read_size;
offs += read_size;
len -= read_size;
}
return true;
}
uint64_t DeviceVDI::GetSize() const
{
return m_disk_size;
}
+30
查看文件
@@ -0,0 +1,30 @@
#pragma once
#include <vector>
#include "Endian.h"
#include "Device.h"
class DeviceVDI : public Device
{
public:
DeviceVDI();
~DeviceVDI();
bool Open(const char *name) override;
void Close() override;
bool Read(void *data, uint64_t offs, uint64_t len) override;
uint64_t GetSize() const override;
private:
uint64_t m_disk_size;
uint32_t m_block_size;
uint32_t m_block_count;
uint32_t m_data_offset;
std::vector<uint32_t> m_block_map;
FILE *m_vdi;
};
+62
查看文件
@@ -0,0 +1,62 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef _WIN32
#include "DeviceWinFile.h"
DeviceWinFile::DeviceWinFile()
{
m_size = 0;
}
DeviceWinFile::~DeviceWinFile()
{
Close();
}
bool DeviceWinFile::Open(const char * name)
{
m_vol.open(name, std::ios::binary);
if (!m_vol.is_open())
return false;
m_vol.seekg(0, std::ios::end);
m_size = m_vol.tellg();
m_vol.seekg(0);
return m_vol.is_open();
}
void DeviceWinFile::Close()
{
m_vol.close();
}
bool DeviceWinFile::Read(void *data, uint64_t offs, uint64_t len)
{
m_vol.seekg(offs);
m_vol.read(reinterpret_cast<char *>(data), len);
// TODO: Fehlerbehandlung ...
return true;
}
#endif
+46
查看文件
@@ -0,0 +1,46 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#ifdef _WIN32
#include <fstream>
#include "Device.h"
class DeviceWinFile : public Device
{
public:
DeviceWinFile();
~DeviceWinFile();
bool Open(const char *name) override;
void Close() override;
bool Read(void *data, uint64_t offs, uint64_t len) override;
uint64_t GetSize() const override { return m_size; }
private:
std::ifstream m_vol;
uint64_t m_size;
};
#endif
+91
查看文件
@@ -0,0 +1,91 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef _WIN32
#include "DeviceWinPhys.h"
DeviceWinPhys::DeviceWinPhys()
{
m_drive = INVALID_HANDLE_VALUE;
m_size = 0;
}
DeviceWinPhys::~DeviceWinPhys()
{
Close();
}
bool DeviceWinPhys::Open(const char *name)
{
TCHAR path[MAX_PATH];
uint8_t buf[0x1000];
DWORD bytes_ret = 0;
// _stprintf_s(path, _T("\\\\.\\PhysicalDrive%d"), disk);
MultiByteToWideChar(CP_UTF8, 0, name, -1, path, MAX_PATH);
m_drive = CreateFile(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, nullptr);
if (m_drive == INVALID_HANDLE_VALUE)
return false;
DeviceIoControl(m_drive, IOCTL_DISK_GET_DRIVE_GEOMETRY_EX, nullptr, 0, &buf, sizeof(buf), &bytes_ret, nullptr);
if (bytes_ret == 0)
return false;
const DISK_GEOMETRY_EX *geom = reinterpret_cast<const DISK_GEOMETRY_EX *>(buf);
m_size = geom->DiskSize.QuadPart;
return true;
}
void DeviceWinPhys::Close()
{
if (m_drive != INVALID_HANDLE_VALUE)
CloseHandle(m_drive);
m_drive = INVALID_HANDLE_VALUE;
m_size = 0;
}
bool DeviceWinPhys::Read(void * data, uint64_t offs, uint64_t len)
{
DWORD read_bytes = 0;
LARGE_INTEGER off;
BOOL rc;
if (m_drive == INVALID_HANDLE_VALUE)
return false;
off.QuadPart = offs;
SetFilePointerEx(m_drive, off, nullptr, FILE_BEGIN);
rc = ReadFile(m_drive, data, len, &read_bytes, nullptr);
return rc == TRUE;
}
uint64_t DeviceWinPhys::GetSize() const
{
return m_size;
}
#endif
+47
查看文件
@@ -0,0 +1,47 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#ifdef _WIN32
#include "Device.h"
// #define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <tchar.h>
class DeviceWinPhys : public Device
{
public:
DeviceWinPhys();
~DeviceWinPhys();
bool Open(const char *name) override;
void Close() override;
bool Read(void *data, uint64_t offs, uint64_t len) override;
uint64_t GetSize() const override;
private:
HANDLE m_drive;
uint64_t m_size;
};
#endif
+518
查看文件
@@ -0,0 +1,518 @@
#include <cstring>
#include <vector>
#include <iostream>
#include "Global.h"
#include "Endian.h"
#include <Crypto/Crypto.h>
#include <Crypto/TripleDes.h>
#include "Util.h"
#include "DiskImageFile.h"
#pragma pack(push, 1)
struct DmgCryptHeaderV1
{
uint8_t uuid[0x10];
be_uint32_t block_size;
be_uint32_t unk_14;
be_uint32_t unk_18;
be_uint32_t unk_1C;
be_uint32_t unk_20;
be_uint32_t unk_24;
be_uint32_t kdf_algorithm;
be_uint32_t kdf_prng_algorithm;
be_uint32_t kdf_iteration_count;
be_uint32_t kdf_salt_len;
uint8_t kdf_salt[0x20];
be_uint32_t unk_58;
be_uint32_t unk_5C;
be_uint32_t unk_60;
be_uint32_t unk_64;
uint8_t unwrap_iv[0x20];
be_uint32_t wrapped_aes_key_len;
uint8_t wrapped_aes_key[0x100];
be_uint32_t unk_18C;
be_uint32_t unk_190;
uint8_t unk_194[0x20];
be_uint32_t wrapped_hmac_sha1_key_len;
uint8_t wrapped_hmac_sha1_key[0x100];
be_uint32_t unk_2B8;
be_uint32_t unk_2BC;
uint8_t unk_2C0[0x20];
be_uint32_t wrapped_integrity_key_len;
uint8_t wrapped_integrity_key[0x100];
be_uint32_t unk_3E8_len;
uint8_t unk_3E8[0x100];
be_uint64_t decrypted_data_length;
be_uint32_t unk_4F0;
char signature[8];
};
static_assert(sizeof(DmgCryptHeaderV1) == 0x4FC, "DmgCryptHeaderV1 invalid length.");
struct DmgCryptHeaderV2
{
char signature[8];
be_uint32_t maybe_version;
be_uint32_t unk_0C;
be_uint32_t unk_10;
be_uint32_t unk_14;
be_uint32_t key_bits;
be_uint32_t unk_1C;
be_uint32_t unk_20;
uint8_t uuid[0x10];
be_uint32_t block_size;
be_uint64_t encrypted_data_length;
be_uint64_t encrypted_data_offset;
be_uint32_t no_of_keys;
};
struct DmgKeyPointer
{
be_uint32_t key_type;
be_uint64_t key_offset;
be_uint64_t key_length;
};
struct DmgKeyData
{
be_uint32_t kdf_algorithm;
be_uint32_t prng_algorithm;
be_uint32_t iteration_count;
be_uint32_t salt_len;
uint8_t salt[0x20];
be_uint32_t blob_enc_iv_size;
uint8_t blob_enc_iv[0x20];
be_uint32_t blob_enc_key_bits;
be_uint32_t blob_enc_algorithm;
be_uint32_t blob_unk_5C;
be_uint32_t blob_enc_mode;
be_uint32_t encr_key_blob_size;
uint8_t encr_key_blob[0x200];
};
#pragma pack(pop)
DiskImageFile::DiskImageFile()
{
m_is_encrypted = false;
m_crypt_offset = 0;
m_crypt_size = 0;
m_crypt_blocksize = 0;
}
DiskImageFile::~DiskImageFile()
{
}
bool DiskImageFile::Open(const char * name)
{
m_image.open(name, std::ios::binary);
return m_image.is_open();
}
void DiskImageFile::Close()
{
m_image.close();
m_crypt_blocksize = 0;
m_crypt_size = 0;
m_crypt_offset = 0;
}
void DiskImageFile::Reset()
{
m_is_encrypted = false;
m_crypt_offset = 0;
m_crypt_size = 0;
m_crypt_blocksize = 0;
memset(m_hmac_key, 0, sizeof(m_hmac_key));
m_aes.CleanUp();
}
bool DiskImageFile::CheckSetupEncryption()
{
char signature[8];
m_image.seekg(0, std::ios::end);
m_is_encrypted = false;
m_crypt_offset = 0;
m_crypt_size = m_image.tellg();
m_image.seekg(-8, std::ios::end);
m_image.read(signature, 8);
if (!memcmp(signature, "cdsaencr", 8))
{
m_is_encrypted = true;
if (!SetupEncryptionV1())
{
m_image.close();
fprintf(stderr, "Error setting up decryption V1.\n");
return false;
}
}
m_image.seekg(0);
m_image.read(signature, 8);
if (!memcmp(signature, "encrcdsa", 8))
{
m_is_encrypted = true;
if (!SetupEncryptionV2())
{
m_image.close();
fprintf(stderr, "Error setting up decryption V2.\n");
return false;
}
}
return true;
}
void DiskImageFile::Read(uint64_t off, void * data, size_t size)
{
if (!m_is_encrypted)
{
m_image.seekg(off);
m_image.read(reinterpret_cast<char *>(data), size);
}
else
{
uint8_t buffer[0x1000];
uint64_t mask = m_crypt_blocksize - 1;
uint32_t blkid;
uint8_t iv[0x14];
size_t rd_len;
uint8_t *bdata = reinterpret_cast<uint8_t *>(data);
if (off & mask)
{
blkid = static_cast<uint32_t>(off / m_crypt_blocksize);
blkid = bswap_be(blkid);
m_image.seekg(m_crypt_offset + (off & ~mask));
m_image.read(reinterpret_cast<char *>(buffer), m_crypt_blocksize);
HMAC_SHA1(m_hmac_key, 0x14, reinterpret_cast<const uint8_t *>(&blkid), sizeof(uint32_t), iv);
m_aes.SetIV(iv);
m_aes.DecryptCBC(buffer, buffer, m_crypt_blocksize);
rd_len = m_crypt_blocksize - (off & mask);
if (rd_len > size)
rd_len = size;
memcpy(bdata, buffer + (off & mask), rd_len);
bdata += rd_len;
off += rd_len;
size -= rd_len;
}
while (size > 0)
{
blkid = static_cast<uint32_t>(off / m_crypt_blocksize);
blkid = bswap_be(blkid);
m_image.seekg(m_crypt_offset + (off & ~mask));
m_image.read(reinterpret_cast<char *>(buffer), m_crypt_blocksize);
HMAC_SHA1(m_hmac_key, 0x14, reinterpret_cast<const uint8_t *>(&blkid), sizeof(uint32_t), iv);
m_aes.SetIV(iv);
m_aes.DecryptCBC(buffer, buffer, m_crypt_blocksize);
rd_len = m_crypt_blocksize;
if (rd_len > size)
rd_len = size;
memcpy(bdata, buffer, rd_len);
bdata += rd_len;
off += rd_len;
size -= rd_len;
}
}
}
bool DiskImageFile::SetupEncryptionV1()
{
DmgCryptHeaderV1 hdr;
TripleDES des;
std::string password;
uint8_t derived_key[0x18];
size_t n;
uint64_t total_size;
static const uint8_t des_iv[8] = { 0x4A, 0xDD, 0xA2, 0x2C, 0x79, 0xE8, 0x21, 0x05 };
// uint8_t aes_key[0x10];
// uint8_t hmac_key[0x14];
// uint8_t integrity_key[0x100];
uint8_t tmp_1[0x100];
uint8_t tmp_2[0x100];
uint8_t tmp_3[0x100];
size_t len;
int64_t hdrsize = sizeof(DmgCryptHeaderV1);
m_image.seekg(-hdrsize, std::ios::end);
total_size = m_image.tellg();
m_image.read(reinterpret_cast<char *>(&hdr), sizeof(hdr));
if (g_debug & Dbg_Crypto)
{
std::cout.flags(std::cout.hex | std::cout.uppercase);
std::cout.fill('0');
}
m_crypt_size = hdr.decrypted_data_length;
m_crypt_blocksize = hdr.block_size;
m_crypt_offset = 0;
m_is_encrypted = true;
std::cout << "Password: ";
GetPassword(password);
PBKDF2_HMAC_SHA1(reinterpret_cast<const uint8_t *>(password.c_str()), password.size(), hdr.kdf_salt, hdr.kdf_salt_len, hdr.kdf_iteration_count, derived_key, 0x18);
if (g_debug & Dbg_Crypto)
{
std::cout << "Salt: " << hexstr(hdr.kdf_salt, hdr.kdf_salt_len) << std::endl;
std::cout << "Iter: " << hdr.kdf_iteration_count << std::endl;
std::cout << "DKey: " << hexstr(derived_key, sizeof(derived_key)) << std::endl;
std::cout << "IV : " << hexstr(hdr.unwrap_iv, sizeof(hdr.unwrap_iv)) << std::endl;
std::cout << "U3E8: " << hexstr(hdr.unk_3E8, hdr.unk_3E8_len) << std::endl;
std::cout << std::endl;
std::cout << "AES Key:" << std::endl;
}
des.SetKey(derived_key);
len = hdr.wrapped_aes_key_len;
des.SetIV(des_iv);
des.DecryptCBC(tmp_1, hdr.wrapped_aes_key, len);
if (g_debug & Dbg_Crypto)
DumpHex(std::cout, tmp_1, len);
if (tmp_1[len - 1] > 0x8)
return false;
len = PkcsUnpad(tmp_1, len);
for (n = 0; n < len; n++)
tmp_2[len - n - 1] = tmp_1[n];
if (g_debug & Dbg_Crypto)
DumpHex(std::cout, tmp_2, len);
des.SetIV(nullptr);
des.DecryptCBC(tmp_3, tmp_2, len);
if (g_debug & Dbg_Crypto)
{
DumpHex(std::cout, tmp_3, len);
std::cout << std::endl;
}
if (tmp_3[len - 1] > 0x8)
return false;
len = PkcsUnpad(tmp_3, len);
// memcpy(aes_key, tmp_3 + 12, 0x10);
m_aes.SetKey(tmp_3 + 12, AES::AES_128);
if (g_debug & Dbg_Crypto)
std::cout << "HMAC SHA1 Key:" << std::endl;
len = hdr.wrapped_hmac_sha1_key_len;
des.SetIV(des_iv);
des.DecryptCBC(tmp_1, hdr.wrapped_hmac_sha1_key, len);
if (g_debug & Dbg_Crypto)
DumpHex(std::cout, tmp_1, len);
if (tmp_1[len - 1] > 0x8)
return false;
len = PkcsUnpad(tmp_1, len);
for (n = 0; n < len; n++)
tmp_2[len - n - 1] = tmp_1[n];
if (g_debug & Dbg_Crypto)
DumpHex(std::cout, tmp_2, len);
des.SetIV(nullptr);
des.DecryptCBC(tmp_3, tmp_2, len);
if (g_debug & Dbg_Crypto)
{
DumpHex(std::cout, tmp_3, len);
std::cout << std::endl;
}
if (tmp_3[len - 1] > 0x8)
return false;
len = PkcsUnpad(tmp_3, len);
// memcpy(hmac_key, tmp_3 + 12, 0x14);
memcpy(m_hmac_key, tmp_3 + 12, 0x14);
if (g_debug & Dbg_Crypto)
std::cout << "Integrity Key:" << std::endl;
len = hdr.wrapped_integrity_key_len;
des.SetIV(des_iv);
des.DecryptCBC(tmp_1, hdr.wrapped_integrity_key, len);
if (g_debug & Dbg_Crypto)
DumpHex(std::cout, tmp_1, len);
if (tmp_1[len - 1] > 0x8)
return false;
len = PkcsUnpad(tmp_1, len);
for (n = 0; n < len; n++)
tmp_2[len - n - 1] = tmp_1[n];
if (g_debug & Dbg_Crypto)
DumpHex(std::cout, tmp_2, len);
des.SetIV(tmp_2);
len -= 8;
des.DecryptCBC(tmp_3, tmp_2 + 8, len);
if (g_debug & Dbg_Crypto)
{
DumpHex(std::cout, tmp_3, len);
std::cout << std::endl;
std::cout << "Unknown Key:" << std::endl;
DumpHex(std::cout, hdr.unk_3E8, hdr.unk_3E8_len);
}
return true;
}
bool DiskImageFile::SetupEncryptionV2()
{
std::vector<uint8_t> data;
std::vector<uint8_t> kdata;
const DmgCryptHeaderV2 *hdr;
const DmgKeyPointer *keyptr;
const DmgKeyData *keydata;
std::string password;
uint32_t no_of_keys;
uint32_t key_id;
uint8_t derived_key[0x18];
uint8_t blob[0x200];
uint32_t blob_len;
TripleDES des;
bool key_ok = false;
data.resize(0x1000);
m_image.seekg(0);
m_image.read(reinterpret_cast<char *>(data.data()), data.size());
hdr = reinterpret_cast<const DmgCryptHeaderV2 *>(data.data());
if (memcmp(hdr->signature, "encrcdsa", 8))
return false;
m_crypt_offset = hdr->encrypted_data_offset;
m_crypt_size = hdr->encrypted_data_length;
m_crypt_blocksize = hdr->block_size;
std::cout << "Encryped DMG detected." << std::endl;
std::cout << "Password: ";
GetPassword(password);
no_of_keys = hdr->no_of_keys;
for (key_id = 0; key_id < no_of_keys; key_id++)
{
keyptr = reinterpret_cast<const DmgKeyPointer *>(data.data() + sizeof(DmgCryptHeaderV2) + key_id * sizeof(DmgKeyPointer));
kdata.resize(keyptr->key_length);
m_image.seekg(keyptr->key_offset.get());
m_image.read(reinterpret_cast<char *>(kdata.data()), kdata.size());
keydata = reinterpret_cast<const DmgKeyData *>(kdata.data());
PBKDF2_HMAC_SHA1(reinterpret_cast<const uint8_t *>(password.c_str()), password.size(), keydata->salt, keydata->salt_len, keydata->iteration_count, derived_key, sizeof(derived_key));
des.SetKey(derived_key);
des.SetIV(keydata->blob_enc_iv);
blob_len = keydata->encr_key_blob_size;
des.DecryptCBC(blob, keydata->encr_key_blob, blob_len);
if (blob[blob_len - 1] < 1 || blob[blob_len - 1] > 8)
continue;
blob_len -= blob[blob_len - 1];
if (g_debug & Dbg_Crypto)
{
std::cout << "Salt: " << hexstr(keydata->salt, keydata->salt_len) << std::endl;
std::cout << "Iter: " << keydata->iteration_count << std::endl;
std::cout << "DKey: " << hexstr(derived_key, sizeof(derived_key)) << std::endl;
std::cout << "Blob: " << hexstr(keydata->encr_key_blob, keydata->encr_key_blob_size) << std::endl;
std::cout << "DBlb: " << hexstr(blob, keydata->encr_key_blob_size) << std::endl;
std::cout << "Key : " << hexstr(blob, hdr->key_bits / 8) << std::endl;
std::cout << "HMAC: " << hexstr(blob + hdr->key_bits / 8, 0x14) << std::endl;
}
if (memcmp(blob + blob_len - 5, "CKIE", 4))
continue;
if (hdr->key_bits == 128)
{
m_aes.SetKey(blob, AES::AES_128);
memcpy(m_hmac_key, blob + 0x10, 0x14);
key_ok = true;
break;
}
else if (hdr->key_bits == 256)
{
m_aes.SetKey(blob, AES::AES_256);
memcpy(m_hmac_key, blob + 0x20, 0x14);
key_ok = true;
break;
}
}
return key_ok;
}
size_t DiskImageFile::PkcsUnpad(const uint8_t *data, size_t size)
{
if (data[size - 1] >= 1 && data[size - 1] <= 8)
size -= data[size - 1];
return size;
}
+39
查看文件
@@ -0,0 +1,39 @@
#pragma once
#include <cstdint>
#include <fstream>
#include <Crypto/Aes.h>
#include "Device.h"
class DiskImageFile
{
public:
DiskImageFile();
~DiskImageFile();
bool Open(const char *name);
void Close();
void Reset();
void Read(uint64_t off, void *data, size_t size);
uint64_t GetContentSize() const { return m_crypt_size; }
bool CheckSetupEncryption();
private:
bool SetupEncryptionV1();
bool SetupEncryptionV2();
size_t PkcsUnpad(const uint8_t *data, size_t size);
std::ifstream m_image;
bool m_is_encrypted;
uint64_t m_crypt_offset;
uint64_t m_crypt_size;
uint32_t m_crypt_blocksize;
uint8_t m_hmac_key[0x14];
AES m_aes;
};
+1515
查看文件
文件差异过大,未显示。 加载差异
+217
查看文件
@@ -0,0 +1,217 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
/*
This file is for handling the different endiannesses used by Apple.
Also helps making the driver run on big-endian architectures.
*/
#pragma once
#include <cstdint>
// This should later be done by some configuration ...
#define APFS_LITTLE_ENDIAN
#undef APFS_BIG_ENDIAN
#ifdef _MSC_VER
// Definitions for Visual Studio
#include <intrin.h>
#define bswap_16(x) _byteswap_ushort(x)
#define bswap_32(x) _byteswap_ulong(x)
#define bswap_64(x) _byteswap_uint64(x)
#define be16toh(x) _byteswap_ushort(x)
#define be32toh(x) _byteswap_ulong(x)
#define be64toh(x) _byteswap_uint64(x)
#define htobe16(x) _byteswap_ushort(x)
#define htobe32(x) _byteswap_ulong(x)
#define htobe64(x) _byteswap_uint64(x)
#define le16toh(x) (x)
#define le32toh(x) (x)
#define le64toh(x) (x)
#define htole16(x) (x)
#define htole32(x) (x)
#define htole64(x) (x)
#define __attribute__(x)
#endif
#ifdef __linux__
// Definitions for Linux
#include <byteswap.h>
#include <endian.h>
#endif
#ifdef __APPLE__
// Definitions for macOS
#include <libkern/OSByteOrder.h>
#define bswap_16(x) _OSSwapInt16(x)
#define bswap_32(x) _OSSwapInt32(x)
#define bswap_64(x) _OSSwapInt64(x)
#define be16toh(x) OSSwapBigToHostInt16(x)
#define be32toh(x) OSSwapBigToHostInt32(x)
#define be64toh(x) OSSwapBigToHostInt64(x)
#define htobe16(x) OSSwapHostToBigInt16(x)
#define htobe32(x) OSSwapHostToBigInt32(x)
#define htobe64(x) OSSwapHostToBigInt64(x)
#define le16toh(x) OSSwapLittleToHostInt16(x)
#define le32toh(x) OSSwapLittleToHostInt32(x)
#define le64toh(x) OSSwapLittleToHostInt64(x)
#define htole16(x) OSSwapHostToLittleInt16(x)
#define htole32(x) OSSwapHostToLittleInt32(x)
#define htole64(x) OSSwapHostToLittleInt64(x)
#endif
#ifdef APFS_LITTLE_ENDIAN
// Swap to/from little endian.
inline int16_t bswap_le(const int16_t &x) { return x; }
inline int32_t bswap_le(const int32_t &x) { return x; }
inline int64_t bswap_le(const int64_t &x) { return x; }
inline uint16_t bswap_le(const uint16_t &x) { return x; }
inline uint32_t bswap_le(const uint32_t &x) { return x; }
inline uint64_t bswap_le(const uint64_t &x) { return x; }
// Swap to/from big endian.
inline int16_t bswap_be(const int16_t& x) { return bswap_16(x); }
inline int32_t bswap_be(const int32_t& x) { return bswap_32(x); }
inline int64_t bswap_be(const int64_t& x) { return bswap_64(x); }
inline uint16_t bswap_be(const uint16_t& x) { return bswap_16(x); }
inline uint32_t bswap_be(const uint32_t& x) { return bswap_32(x); }
inline uint64_t bswap_be(const uint64_t& x) { return bswap_64(x); }
typedef uint8_t le_uint8_t;
typedef uint16_t le_uint16_t;
typedef uint32_t le_uint32_t;
typedef uint64_t le_uint64_t;
typedef int8_t le_int8_t;
typedef int16_t le_int16_t;
typedef int32_t le_int32_t;
typedef int64_t le_int64_t;
typedef uint8_t be_uint8_t;
struct be_uint16_t {
void operator=(uint16_t v) { val = bswap_16(v); }
operator uint16_t() const { return bswap_16(val); }
uint16_t get() const { return bswap_16(val); }
private:
uint16_t val;
} __attribute__((packed));
struct be_uint32_t {
void operator=(uint32_t v) { val = bswap_32(v); }
operator uint32_t() const { return bswap_32(val); }
uint32_t get() const { return bswap_32(val); }
private:
uint32_t val;
} __attribute__((packed));
struct be_uint64_t {
void operator=(uint64_t v) { val = bswap_64(v); }
operator uint64_t() const { return bswap_64(val); }
uint64_t get() const { return bswap_64(val); }
private:
uint64_t val;
} __attribute__((packed));
#endif
#ifdef APFS_BIG_ENDIAN
// Swap to/from little endian.
inline int16_t bswap_le(const int16_t &x) { return bswap_16(x); }
inline int32_t bswap_le(const int32_t &x) { return bswap_32(x); }
inline int64_t bswap_le(const int64_t &x) { return bswap_64(x); }
inline uint16_t bswap_le(const uint16_t &x) { return bswap_16(x); }
inline uint32_t bswap_le(const uint32_t &x) { return bswap_32(x); }
inline uint64_t bswap_le(const uint64_t &x) { return bswap_64(x); }
// Swap to/from big endian.
inline int16_t bswap_be(const int16_t &x) { return x; }
inline int32_t bswap_be(const int32_t &x) { return x; }
inline int64_t bswap_be(const int64_t &x) { return x; }
inline uint16_t bswap_be(const uint16_t &x) { return x; }
inline uint32_t bswap_be(const uint32_t &x) { return x; }
inline uint64_t bswap_be(const uint64_t &x) { return x; }
typedef uint8_t le_uint8_t;
struct le_uint16_t {
void operator=(uint16_t v) { val = bswap_16(v); }
operator uint16_t() const { return bswap_16(val); }
uint16_t get() const { return bswap_16(val); }
private:
uint16_t val;
} __attribute__((packed));
struct le_uint32_t {
void operator=(uint32_t v) { val = bswap_32(v); }
operator uint32_t() const { return bswap_32(val); }
uint32_t get() const { return bswap_32(val); }
private:
uint32_t val;
} __attribute__((packed));
struct le_uint64_t {
void operator=(uint64_t v) { val = bswap_64(v); }
operator uint64_t() const { return bswap_64(val); }
uint64_t get() const { return bswap_64(val); }
private:
uint64_t val;
} __attribute__((packed));
struct le_int64_t {
void operator=(int64_t v) { val = bswap_64(v); }
operator int64_t() const { return bswap_64(val); }
int64_t get() const { return bswap_64(val); }
private:
int64_t val;
} __attribute__((packed));
typedef uint8_t be_uint8_t;
typedef uint16_t be_uint16_t;
typedef uint32_t be_uint32_t;
typedef uint64_t be_uint64_t;
#endif
#if 0
inline int8_t bswap_le(const int8_t &x) { return x; }
inline int8_t bswap_be(const int8_t &x) { return x; }
inline uint8_t bswap_le(const uint8_t &x) { return x; }
inline uint8_t bswap_be(const uint8_t &x) { return x; }
// Big-endian template that does automatic swapping if necessary.
template<typename T>
struct be
{
public:
void operator=(T v) { val = bswap_be(v); }
operator T() const { return bswap_be(val); }
T get() const { return bswap_be(val); }
private:
T val;
};
// Little-endian template that does automatic swapping if necessary.
template<typename T>
struct le
{
public:
void operator=(T v) { val = bswap_le(v); }
operator T() const { return bswap_le(val); }
T get() const { return bswap_le(val); }
private:
T val;
};
#endif
#ifdef _MSC_VER
#undef __attribute__
#endif
+36
查看文件
@@ -0,0 +1,36 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <cstdint>
// Debug level - defined in ApfsContainer.cpp
extern int g_debug;
// Lax mode - defined in ApfsContainer.cpp
extern bool g_lax;
enum DbgFlags
{
Dbg_Errors = 1,
Dbg_Info = 2,
Dbg_Dir = 4,
Dbg_Cmpfs = 8,
Dbg_Crypto = 16
};
+200
查看文件
@@ -0,0 +1,200 @@
#include <cstdio>
#include <cstring>
#include <cinttypes>
#include "Device.h"
#include "GptPartitionMap.h"
typedef uint8_t PM_GUID[0x10];
#pragma pack(1)
struct PMAP_GptHeader
{
le_uint64_t Signature;
le_uint32_t Revision;
le_uint32_t HeaderSize;
le_uint32_t HeaderCRC32;
le_uint32_t Reserved;
le_uint64_t MyLBA;
le_uint64_t AlternateLBA;
le_uint64_t FirstUsableLBA;
le_uint64_t LastUsableLBA;
PM_GUID DiskGUID;
le_uint64_t PartitionEntryLBA;
le_uint32_t NumberOfPartitionEntries;
le_uint32_t SizeOfPartitionEntry;
le_uint32_t PartitionEntryArrayCRC32;
};
struct PMAP_Entry
{
PM_GUID PartitionTypeGUID;
PM_GUID UniquePartitionGUID;
le_uint64_t StartingLBA;
le_uint64_t EndingLBA;
le_uint64_t Attributes;
le_uint16_t PartitionName[36];
};
static_assert(sizeof(PMAP_GptHeader) == 92, "PMAP GPT-Header wrong size");
static_assert(sizeof(PMAP_Entry) == 128, "PMAP Entry wrong size");
#pragma pack()
static const PM_GUID partitiontype_apfs = { 0xEF, 0x57, 0x34, 0x7C, 0x00, 0x00, 0xAA, 0x11, 0xAA, 0x11, 0x00, 0x30, 0x65, 0x43, 0xEC, 0xAC };
static void PrintGUID(const PM_GUID &guid)
{
printf("%02X%02X%02X%02X-", guid[3], guid[2], guid[1], guid[0]);
printf("%02X%02X-", guid[5], guid[4]);
printf("%02X%02X-", guid[7], guid[6]);
printf("%02X%02X-", guid[9], guid[8]);
printf("%02X%02X%02X%02X%02X%02X", guid[10], guid[11], guid[12], guid[13], guid[14], guid[15]);
}
GptPartitionMap::GptPartitionMap() : m_crc(true)
{
m_hdr = nullptr;
m_map = nullptr;
m_sector_size = 0x200;
}
bool GptPartitionMap::LoadAndVerify(Device & dev)
{
PMAP_GptHeader *hdr;
m_hdr = nullptr;
m_map = nullptr;
m_entry_data.clear();
m_hdr_data.clear();
m_sector_size = dev.GetSectorSize();
m_hdr_data.resize(m_sector_size);
dev.Read(m_hdr_data.data(), m_sector_size, m_sector_size);
hdr = reinterpret_cast<PMAP_GptHeader *>(m_hdr_data.data());
if (hdr->Signature != 0x5452415020494645) {
m_hdr_data.resize(0x1000);
dev.Read(m_hdr_data.data(), 0x1000, 0x1000);
hdr = reinterpret_cast<PMAP_GptHeader *>(m_hdr_data.data());
if (hdr->Signature != 0x5452415020494645)
return false;
m_sector_size = 0x1000;
}
if (hdr->Revision != 0x00010000)
return false;
if (hdr->HeaderSize > m_sector_size)
return false;
if (hdr->SizeOfPartitionEntry != 0x80)
return false;
uint32_t hdr_crc;
uint32_t calc_crc;
hdr_crc = hdr->HeaderCRC32;
hdr->HeaderCRC32 = 0;
m_crc.SetCRC(0xFFFFFFFF);
m_crc.Calc(m_hdr_data.data(), hdr->HeaderSize);
calc_crc = m_crc.GetCRC() ^ 0xFFFFFFFF;
if (calc_crc != hdr_crc) {
m_hdr_data.clear();
return false;
}
size_t mapsize;
mapsize = hdr->NumberOfPartitionEntries * hdr->SizeOfPartitionEntry;
mapsize = (mapsize + m_sector_size - 1) & ~(static_cast<size_t>(m_sector_size) - 1);
m_entry_data.resize(mapsize);
dev.Read(m_entry_data.data(), m_sector_size * hdr->PartitionEntryLBA, m_entry_data.size());
m_crc.SetCRC(0xFFFFFFFF);
m_crc.Calc(m_entry_data.data(), hdr->SizeOfPartitionEntry * hdr->NumberOfPartitionEntries);
calc_crc = m_crc.GetCRC() ^ 0xFFFFFFFF;
if (calc_crc != hdr->PartitionEntryArrayCRC32) {
m_entry_data.clear();
m_hdr_data.clear();
return false;
}
m_hdr = reinterpret_cast<const PMAP_GptHeader *>(m_hdr_data.data());
m_map = reinterpret_cast<const PMAP_Entry *>(m_entry_data.data());
return true;
}
int GptPartitionMap::FindFirstAPFSPartition()
{
if (!m_hdr || !m_map)
return -1;
unsigned int k;
int rc = -1;
for (k = 0; k < m_hdr->NumberOfPartitionEntries; k++)
{
if (m_map[k].StartingLBA == 0 && m_map[k].EndingLBA == 0)
break;
if (!memcmp(m_map[k].PartitionTypeGUID, partitiontype_apfs, sizeof(PM_GUID)))
{
rc = k;
break;
}
}
return rc;
}
bool GptPartitionMap::GetPartitionOffsetAndSize(int partnum, uint64_t & offset, uint64_t & size)
{
if (!m_hdr || !m_map)
return false;
offset = m_map[partnum].StartingLBA * m_sector_size;
size = (m_map[partnum].EndingLBA - m_map[partnum].StartingLBA + 1) * m_sector_size;
return true;
}
void GptPartitionMap::ListEntries()
{
if (!m_hdr || !m_map)
return;
size_t k;
size_t n;
for (k = 0; k < m_hdr->NumberOfPartitionEntries; k++)
{
const PMAP_Entry &e = m_map[k];
if (e.StartingLBA == 0 && e.EndingLBA == 0)
break;
PrintGUID(e.PartitionTypeGUID);
printf(" ");
PrintGUID(e.UniquePartitionGUID);
printf(" %016" PRIX64 " %016" PRIX64 " ", e.StartingLBA, e.EndingLBA);
printf("%016" PRIX64 " ", e.Attributes);
for (n = 0; (n < 36) && (e.PartitionName[n] != 0); n++)
printf("%c", e.PartitionName[n]);
printf("\n");
}
}
+35
查看文件
@@ -0,0 +1,35 @@
#pragma once
#include <cstdint>
#include <vector>
#include "Endian.h"
#include "Crc32.h"
class Device;
struct PMAP_GptHeader;
struct PMAP_Entry;
class GptPartitionMap
{
public:
GptPartitionMap();
bool LoadAndVerify(Device &dev);
int FindFirstAPFSPartition();
bool GetPartitionOffsetAndSize(int partnum, uint64_t &offset, uint64_t &size);
void ListEntries();
private:
Crc32 m_crc;
std::vector<uint8_t> m_hdr_data;
std::vector<uint8_t> m_entry_data;
const PMAP_GptHeader *m_hdr;
const PMAP_Entry *m_map;
unsigned int m_sector_size;
};
+741
查看文件
@@ -0,0 +1,741 @@
#include <cstring>
#include <iomanip>
#include <iostream>
#include <sstream>
#include "ApfsContainer.h"
#include <Crypto/Asn1Der.h>
#include <Crypto/AesXts.h>
#include <Crypto/Sha256.h>
#include <Crypto/Crypto.h>
#include "Util.h"
#include "BlockDumper.h"
#include "KeyMgmt.h"
struct bagdata_t
{
const uint8_t *data;
size_t size;
};
struct key_info_t
{
uint32_t flags;
uint8_t unk_04;
uint8_t unk_05;
uint8_t uuid[16];
};
struct key_header_t
{
uint64_t unk_0;
apfs_uuid_t uuid;
key_info_t info;
};
struct kek_entry_t
{
key_header_t hdr;
uint8_t wrapped_kek[0x28];
uint64_t iterations;
uint8_t salt[0x10];
};
struct vek_entry_t
{
key_header_t hdr;
uint8_t wrapped_vek[0x28];
};
Keybag::Keybag()
{
m_kl = nullptr;
}
Keybag::~Keybag()
{
}
bool Keybag::Init(const media_keybag_t * mk, size_t size)
{
(void)size;
if (mk->mk_locker.kl_version != 2)
return false;
const uint8_t *data = reinterpret_cast<const uint8_t *>(&mk->mk_locker);
m_data.assign(data, data + mk->mk_locker.kl_nbytes);
m_kl = reinterpret_cast<kb_locker_t *>(m_data.data());
return true;
}
size_t Keybag::GetKeyCnt()
{
if (m_kl)
return m_kl->kl_nkeys;
else
return 0;
}
const keybag_entry_t * Keybag::GetKey(size_t nr)
{
if (!m_kl)
return nullptr;
if (nr >= m_kl->kl_nkeys)
return nullptr;
const uint8_t *ptr;
const keybag_entry_t *kb;
size_t len;
size_t k;
ptr = m_kl->kl_entries;
for (k = 0; k < nr; k++)
{
kb = reinterpret_cast<const keybag_entry_t *>(ptr);
len = (kb->ke_keylen + sizeof(keybag_entry_t) + 0x0F) & ~0xF;
ptr += len;
}
kb = reinterpret_cast<const keybag_entry_t *>(ptr);
return kb;
}
const keybag_entry_t * Keybag::FindKey(const apfs_uuid_t & uuid, uint16_t type)
{
if (!m_kl)
return nullptr;
const uint8_t *ptr;
const keybag_entry_t *kb;
size_t len;
size_t k;
ptr = m_kl->kl_entries;
for (k = 0; k < m_kl->kl_nkeys; k++)
{
kb = reinterpret_cast<const keybag_entry_t *>(ptr);
if (memcmp(uuid, kb->ke_uuid, sizeof(apfs_uuid_t)) == 0 && kb->ke_tag == type)
return kb;
len = (kb->ke_keylen + sizeof(keybag_entry_t) + 0x0F) & ~0xF;
ptr += len;
}
return nullptr;
}
#undef DUMP_RAW_KEYS
void Keybag::dump(std::ostream &st, Keybag *cbag, const apfs_uuid_t &vuuid)
{
using namespace std;
(void)vuuid;
size_t s;
size_t k;
const keybag_entry_t *ke;
bagdata_t bd;
const char *typestr;
st << "Dumping Keybag (" << (cbag ? "recs" : "keys") << ")" << endl;
st << endl;
if (m_data.empty() || !m_kl)
{
st << "Keybag is empty." << endl;
return;
}
#ifdef DUMP_RAW_KEYS
DumpHex(st, m_data.data(), m_data.size());
st << endl;
#endif
st << "Version : " << setw(4) << m_kl->kl_version << endl;
st << "Keys : " << setw(4) << m_kl->kl_nkeys << endl;
st << "Bytes : " << setw(8) << m_kl->kl_nbytes << endl;
st << endl;
s = GetKeyCnt();
for (k = 0; k < s; k++)
{
ke = GetKey(k);
typestr = "!!! Unknown !!!";
if (!cbag)
{
switch (ke->ke_tag)
{
case KB_TAG_VOLUME_KEY: typestr = "VEK"; break;
case KB_TAG_VOLUME_UNLOCK_RECORDS: typestr = "Keybag Ref"; break;
default: break;
}
}
else
{
switch (ke->ke_tag)
{
case KB_TAG_VOLUME_UNLOCK_RECORDS: typestr = "KEK"; break;
case KB_TAG_VOLUME_PASSPHRASE_HINT: typestr = "Password Hint"; break;
default: break;
}
}
st << "Key " << k << ":" << endl;
st << "UUID : " << uuidstr(ke->ke_uuid) << endl;
st << "Type : " << setw(4) << ke->ke_tag << " [" << typestr << "]" << endl;
st << "Length : " << setw(4) << ke->ke_keylen << endl;
/* Skip the ke->_padding_ */
// DumpHex(st, kd.data.data, kd.data.size);
#ifdef DUMP_RAW_KEYS
DumpHex(st, ke->ke_keydata, ke->ke_keylen);
st << endl;
#endif
if (!cbag)
{
switch (ke->ke_tag)
{
case 2:
der_dump(ke->ke_keydata, ke->ke_keylen);
#if 0
bd.data = ke->ke_keydata;
bd.size = ke->ke_keylen;
if (KeyManager::DecodeBlobHeader(bhdr, bd))
{
st << "[Blob Header]" << endl;
st << "Unk 80 : " << bhdr.unk_80 << endl;
st << "HMAC : " << hexstr(bhdr.hmac, sizeof(bhdr.hmac)) << endl;
st << "Salt : " << hexstr(bhdr.salt, sizeof(bhdr.salt)) << endl;
#ifdef DUMP_RAW_KEYS
st << "Data :" << endl;
DumpHex(st, bhdr.blob.data, bhdr.blob.size);
#endif
st << endl;
vek_blob_t vek;
if (KeyManager::DecodeVEKBlob(vek, bhdr.blob))
{
st << "[VEK]" << endl;
st << "Unk 80 : " << vek.unk_80 << endl;
st << "UUID : " << uuidstr(vek.uuid) << endl;
st << "Unk 82 : " << setw(8) << vek.unk_82.unk_00 << ' ' << setw(4) << vek.unk_82.unk_04 << ' ';
st << setw(2) << static_cast<int>(vek.unk_82.unk_06) << ' ' << setw(2) << static_cast<int>(vek.unk_82.unk_07) << endl;
st << "VEK Wrpd: " << hexstr(vek.wrapped_vek, sizeof(vek.wrapped_vek)) << endl;
st << endl;
}
else
{
st << "Invalid VEK Blob!!!" << endl;
}
}
else
{
st << "Invalid BLOB Header!!!" << endl;
}
#endif
break;
case 3:
{
const prange_t *pr = reinterpret_cast<const prange_t *>(ke->ke_keydata);
st << "Block : " << setw(16) << pr->pr_start_addr << endl;
st << "Count : " << setw(16) << pr->pr_block_count << endl;
}
break;
default:
st << "Unknown Type !!!" << endl;
break;
}
}
else
{
switch (ke->ke_tag)
{
case 3:
der_dump(ke->ke_keydata, ke->ke_keylen);
#if 0
bd.data = ke->ke_keydata;
bd.size = ke->ke_keylen;
if (KeyManager::DecodeBlobHeader(bhdr, bd))
{
st << "[Blob Header]" << endl;
st << "Unk 80 : " << bhdr.unk_80 << endl;
st << "HMAC : " << hexstr(bhdr.hmac, sizeof(bhdr.hmac)) << endl;
st << "Salt : " << hexstr(bhdr.salt, sizeof(bhdr.salt)) << endl;
#ifdef DUMP_RAW_KEYS
st << "Data :" << endl;
DumpHex(st, bhdr.blob.data, bhdr.blob.size);
#endif
st << endl;
kek_blob_t kek;
if (KeyManager::DecodeKEKBlob(kek, bhdr.blob))
{
st << "[KEK]" << endl;
st << "Unk 80 : " << kek.unk_80 << endl;
st << "UUID : " << uuidstr(kek.uuid) << endl;
st << "Unk 82 : " << setw(8) << kek.unk_82.unk_00 << ' ' << setw(4) << kek.unk_82.unk_04 << ' ';
st << setw(2) << static_cast<int>(kek.unk_82.unk_06) << ' ' << setw(2) << static_cast<int>(kek.unk_82.unk_07) << endl;
st << "KEK Wrpd: " << hexstr(kek.wrapped_kek, sizeof(kek.wrapped_kek)) << endl;
st << "Iterat's: " << dec << kek.iterations << hex << endl;
st << "Salt : " << hexstr(kek.salt, sizeof(kek.salt)) << endl;
st << endl;
#if 0 // Test decryption of keys
string pw;
uint8_t dk[0x20] = { 0 };
uint8_t kekk[0x20] = { 0 };
uint64_t iv;
cout << "Enter Password for KEK " << uuidstr(kd.header->uuid) << endl;
GetPassword(pw);
st << "[Decryption Check]" << endl;
PBKDF2_HMAC_SHA256(reinterpret_cast<const uint8_t *>(pw.c_str()), pw.size(), kek.salt, sizeof(kek.salt), kek.iterations, dk, sizeof(dk));
st << "PW DKey : " << hexstr(dk, sizeof(dk)) << endl;
if (kek.unk_82.unk_00 == 0 || kek.unk_82.unk_00 == 0x10)
Rfc3394_KeyUnwrap(kekk, kek.wrapped_kek, 0x20, dk, AES::AES_256, &iv);
else if (kek.unk_82.unk_00 == 2)
Rfc3394_KeyUnwrap(kekk, kek.wrapped_kek, 0x10, dk, AES::AES_128, &iv);
else
{
st << "kek/82/00 = " << kek.unk_82.unk_00 << " ???" << endl;
iv = 0;
}
st << "KEK Dec : " << hexstr(kekk, sizeof(kekk)) << endl;
st << "IV KEK : " << setw(16) << iv << " [" << (iv == 0xA6A6A6A6A6A6A6A6 ? "Correct" : "!!!INCORRECT!!!") << "]" << endl;
if (iv != 0xA6A6A6A6A6A6A6A6)
cerr << "Warning: Wrong password." << endl;
else
{
vek_blob_t vek;
uint8_t vekk[0x20] = { 0 };
if (cbag->FindKey(vuuid, 2, kd))
{
if (KeyManager::DecodeBlobHeader(bhdr, kd.data))
{
if (KeyManager::DecodeVEKBlob(vek, bhdr.blob))
{
if (vek.unk_82.unk_00 == 0)
Rfc3394_KeyUnwrap(vekk, vek.wrapped_vek, 0x20, kekk, AES::AES_256, &iv);
else if (vek.unk_82.unk_00 == 2)
Rfc3394_KeyUnwrap(vekk, vek.wrapped_vek, 0x10, kekk, AES::AES_128, &iv);
else
{
st << "vek/82/00 = " << vek.unk_82.unk_00 << " ???" << endl;
iv = 0;
}
st << "VEK Dec : " << hexstr(vekk, sizeof(vekk)) << endl;
st << "IV VEK : " << setw(16) << iv << " [" << (iv == 0xA6A6A6A6A6A6A6A6 ? "Correct" : "!!!INCORRECT!!!") << "]" << endl;
if (iv != 0xA6A6A6A6A6A6A6A6)
cerr << "Error: Password correct, but KEK can't decrypt VEK ..." << endl;
}
}
}
}
#endif
}
else
{
st << "Invalid KEK Blob!!!" << endl;
}
}
else
{
st << "Invalid BLOB Header!!!" << endl;
}
#endif
break;
case 4:
st << "Hint : " << string(reinterpret_cast<const char *>(ke->ke_keydata), ke->ke_keylen) << endl;
break;
}
}
st << endl;
}
st << endl;
}
KeyManager::KeyManager(ApfsContainer& container) : m_container(container)
{
memset(m_container_uuid, 0, sizeof(m_container_uuid));
m_is_valid = false;
m_is_unencrypted = false;
}
KeyManager::~KeyManager()
{
}
bool KeyManager::Init(uint64_t block, uint64_t blockcnt, const apfs_uuid_t& container_uuid)
{
bool rc;
rc = LoadKeybag(m_container_bag, 0x6B657973, block, blockcnt, container_uuid);
if (rc)
memcpy(m_container_uuid, container_uuid, sizeof(apfs_uuid_t));
else
memset(m_container_uuid, 0, sizeof(m_container_uuid));
m_is_valid = rc;
return rc;
}
bool KeyManager::GetPasswordHint(std::string& hint, const apfs_uuid_t& volume_uuid)
{
const keybag_entry_t *ke;
ke = m_container_bag.FindKey(volume_uuid, KB_TAG_VOLUME_UNLOCK_RECORDS);
if (!ke)
return false;
if (ke->ke_keylen != sizeof(prange_t))
return false;
const prange_t *pr = reinterpret_cast<const prange_t *>(ke->ke_keydata);
Keybag recs_bag;
if (!LoadKeybag(recs_bag, APFS_VOL_KEYBAG_OBJ, pr->pr_start_addr, pr->pr_block_count, volume_uuid))
return false;
ke = recs_bag.FindKey(volume_uuid, KB_TAG_VOLUME_PASSPHRASE_HINT);
if (!ke)
return false;
hint.assign(reinterpret_cast<const char *>(ke->ke_keydata), ke->ke_keylen);
return true;
}
bool KeyManager::GetVolumeKey(uint8_t* vek, const apfs_uuid_t& volume_uuid, const char* password)
{
const keybag_entry_t *ke_recs;
if (g_debug & Dbg_Crypto)
{
std::cout.setf(std::ios::hex | std::ios::uppercase);
std::cout.fill('0');
m_container_bag.dump(std::cout, nullptr, volume_uuid);
}
ke_recs = m_container_bag.FindKey(volume_uuid, 3);
if (!ke_recs)
return false;
if (ke_recs->ke_keylen != sizeof(prange_t))
return false;
const prange_t *pr = reinterpret_cast<const prange_t *>(ke_recs->ke_keydata);
Keybag recs_bag;
if (!LoadKeybag(recs_bag, APFS_VOL_KEYBAG_OBJ, pr->pr_start_addr, pr->pr_block_count, volume_uuid))
return false;
if (g_debug & Dbg_Crypto)
recs_bag.dump(std::cout, &m_container_bag, volume_uuid);
uint8_t dk[0x20] = {};
uint8_t kek[0x20] = {};
uint64_t iv;
bool rc = false;
const keybag_entry_t *ke_kek;
const keybag_entry_t *ke_vek;
bagdata_t bd;
kek_entry_t keke;
vek_entry_t veke;
AES::Mode kek_mode = AES::AES_256;
int cnt = recs_bag.GetKeyCnt();
int k;
// Check all KEKs for any valid KEK.
for (k = 0; k < cnt; k++)
{
ke_kek = recs_bag.GetKey(k);
if (!ke_kek)
continue;
if (ke_kek->ke_tag != KB_TAG_VOLUME_UNLOCK_RECORDS)
continue;
if (!DecodeKEK(keke, ke_kek->ke_keydata, ke_kek->ke_keydata + ke_kek->ke_keylen))
continue;
PBKDF2_HMAC_SHA256(reinterpret_cast<const uint8_t *>(password), strlen(password), keke.salt, sizeof(keke.salt), keke.iterations, dk, sizeof(dk));
// There are more variants here ... 1 = hw_crypt
if (keke.hdr.info.flags & 2) {
rc = Rfc3394_KeyUnwrap(kek, keke.wrapped_kek, 0x10, dk, AES::AES_128, &iv);
kek_mode = AES::AES_128;
} else {
rc = Rfc3394_KeyUnwrap(kek, keke.wrapped_kek, 0x20, dk, AES::AES_256, &iv);
kek_mode = AES::AES_256;
}
if (g_debug & Dbg_Crypto)
{
std::cout << "!!! WARNING !!!" << std::endl;
std::cout << "Do not post the output \"PW Key\", \"KEK\" or \"VEK\" on the internet !!!" << std::endl;
std::cout << "These contain your encryption keys." << std::endl << std::endl;
std::cout << "PW Key : " << hexstr(dk, sizeof(dk)) << std::endl;
std::cout << "KEK Wrpd: " << hexstr(keke.wrapped_kek, sizeof(keke.wrapped_kek)) << std::endl;
std::cout << "KEK : " << hexstr(kek, 0x20) << std::endl;
std::cout << "KEK IV : " << std::setw(16) << iv << std::endl;
std::cout << std::endl;
}
if (rc)
break;
}
if (!rc)
{
if (g_debug & Dbg_Crypto)
std::cout << "Password doesn't work for any key." << std::endl;
return false;
}
ke_vek = m_container_bag.FindKey(volume_uuid, KB_TAG_VOLUME_KEY);
if (!ke_vek)
return false;
if (!DecodeVEK(veke, ke_vek->ke_keydata, ke_vek->ke_keydata + ke_vek->ke_keylen))
return false;
memset(vek, 0, 0x20);
// TODO 1
if (veke.hdr.info.flags & 2) {
// AES-128. This method is used for FileVault and CoreStorage encrypted
// volumes that have been converted to APFS.
rc = Rfc3394_KeyUnwrap(vek, veke.wrapped_vek, 0x10, kek, kek_mode, &iv);
if (rc)
{
SHA256 sha;
uint8_t sha_result[0x20];
sha.Init();
// Use (VEK || vek_blob.uuid), then SHA256, then take the first 16 bytes
sha.Update(vek, 0x10);
sha.Update(veke.hdr.uuid, 0x10);
sha.Final(sha_result);
memcpy(vek + 0x10, sha_result, 0x10);
}
} else {
// AES-256. This method is used for wrapping the whole XTS-AES key,
// and applies to non-FileVault encrypted APFS volumes.
rc = Rfc3394_KeyUnwrap(vek, veke.wrapped_vek, 0x20, kek, kek_mode, &iv);
}
if (g_debug & Dbg_Crypto)
{
std::cout << "VEK Wrpd: " << hexstr(veke.wrapped_vek, 0x28) << std::endl;
std::cout << "VEK : " << hexstr(vek, 0x20) << std::endl;
std::cout << "VEK IV : " << std::setw(16) << iv << std::endl;
}
return rc;
}
void KeyManager::dump(std::ostream &st)
{
size_t k;
size_t s;
const keybag_entry_t *ke;
apfs_uuid_t dummy_uuid;
std::ios::fmtflags fl = st.setf(st.hex | st.uppercase);
char ch = st.fill('0');
memset(dummy_uuid, 0, sizeof(dummy_uuid));
m_container_bag.dump(st, nullptr, dummy_uuid);
s = m_container_bag.GetKeyCnt();
for (k = 0; k < s; k++)
{
ke = m_container_bag.GetKey(k);
if (!ke)
continue;
if ((ke->ke_tag == KB_TAG_VOLUME_UNLOCK_RECORDS) && (ke->ke_keylen == sizeof(prange_t)))
{
st << std::endl;
st << "---------------------------------------------------------------------------------------------------------------------------" << std::endl;
st << std::endl;
const prange_t *pr = reinterpret_cast<const prange_t *>(ke->ke_keydata);
Keybag recs_bag;
if (LoadKeybag(recs_bag, APFS_VOL_KEYBAG_OBJ, pr->pr_start_addr, pr->pr_block_count, ke->ke_uuid))
recs_bag.dump(st, &m_container_bag, ke->ke_uuid);
}
}
st << std::endl;
st << "===========================================================================================================================" << std::endl;
st << std::endl;
st.fill(ch);
st.setf(fl);
}
bool KeyManager::LoadKeybag(Keybag& bag, uint32_t type, uint64_t block, uint64_t blockcnt, const apfs_uuid_t& uuid)
{
std::vector<uint8_t> data;
size_t k;
const size_t blocksize = m_container.GetBlocksize();
if (g_debug & Dbg_Crypto)
std::cout << "starting LoadKeybag @ " << std::hex << block << std::endl;
data.resize(blockcnt * blocksize);
const media_keybag_t *mk = reinterpret_cast<const media_keybag_t *>(data.data());
m_container.ReadBlocks(data.data(), block, blockcnt);
if (mk->mk_obj.o_type == type)
m_is_unencrypted = true;
else
DecryptBlocks(data.data(), block, blockcnt, uuid);
for (k = 0; k < blockcnt; k++)
{
if (!VerifyBlock(data.data(), blockcnt * blocksize))
return false;
}
if (g_debug & Dbg_Crypto)
std::cout << " all blocks verified" << std::endl;
if (mk->mk_obj.o_type != type)
{
if (g_debug & Dbg_Errors)
{
std::cout << "Keybag block types not matching: " << mk->mk_obj.o_type << ", expected " << type << std::endl;
DumpHex(std::cout, data.data(), data.size());
}
return false;
}
bag.Init(mk, data.size()); // TODO: This only works with one block ...
return true;
}
void KeyManager::DecryptBlocks(uint8_t* data, uint64_t block, uint64_t cnt, const uint8_t* key)
{
AesXts xts;
size_t k;
size_t size;
uint64_t uno;
uint64_t cs_factor = m_container.GetBlocksize() / 0x200;
xts.SetKey(key, key);
uno = block * cs_factor;
size = m_container.GetBlocksize() * cnt;
for (k = 0; k < size; k += 0x200)
{
xts.Decrypt(data + k, data + k, 0x200, uno);
uno++;
}
}
const uint8_t* KeyManager::DecodeKeyHeader(key_header_t& hdr, const uint8_t*& end, const uint8_t* der, const uint8_t* der_end)
{
static const uint8_t blob_cookie[6] = { 0x01, 0x16, 0x20, 0x17, 0x15, 0x05 };
const uint8_t* body_end;
uint64_t hmac_0;
uint8_t hmac_hash[32];
uint8_t hmac_salt[8];
size_t info_len;
SHA256 sha256;
uint8_t hmac_key[32];
uint8_t hmac_res[40];
der = der_decode_sequence_tl(body_end, der, der_end);
if (der == nullptr) return nullptr;
der = der_decode_uint64(0x8000000000000000U, hmac_0, der, body_end);
der = der_decode_octet_string_copy(0x8000000000000001U, hmac_hash, 32, der, body_end);
der = der_decode_octet_string_copy(0x8000000000000002U, hmac_salt, 8, der, body_end);
if (der == nullptr) {
log_error("Keybag: Parsing key header failed.\n");
return nullptr;
}
sha256.Init();
sha256.Update(blob_cookie, 6);
sha256.Update(hmac_salt, 8);
sha256.Final(hmac_key);
HMAC_SHA256(hmac_key, 32, der, body_end - der, hmac_res);
if (memcmp(hmac_hash, hmac_res, 32) != 0)
log_error("Keybag: HMAC verification failed.\n");
// Ignoring for now ...
der = der_decode_constructed_tl(0xA000000000000003U, end, der, body_end);
der = der_decode_uint64(0x8000000000000000U, hdr.unk_0, der, end);
der = der_decode_octet_string_copy(0x8000000000000001U, hdr.uuid, 16, der, end);
der = der_decode_tl(0x8000000000000002U, info_len, der, end);
if (info_len > 0x16) return nullptr;
memcpy(&hdr.info, der, info_len);
der += info_len;
hdr.info.flags = bswap_le(hdr.info.flags);
return der;
}
bool KeyManager::DecodeKEK(kek_entry_t& kek, const uint8_t* der, const uint8_t* der_end)
{
const uint8_t* end = nullptr;
memset(&kek, 0, sizeof(kek));
der = DecodeKeyHeader(kek.hdr, end, der, der_end);
if (der == nullptr) return false;
der = der_decode_octet_string_copy(0x8000000000000003U, kek.wrapped_kek, 40, der, end);
der = der_decode_uint64(0x8000000000000004U, kek.iterations, der, end);
der = der_decode_octet_string_copy(0x8000000000000005U, kek.salt, 16, der, end);
return der != nullptr;
}
bool KeyManager::DecodeVEK(vek_entry_t& vek, const uint8_t* der, const uint8_t* der_end)
{
const uint8_t* end = nullptr;
memset(&vek, 0, sizeof(vek));
der = DecodeKeyHeader(vek.hdr, end, der, der_end);
if (der == nullptr) return false;
der = der_decode_octet_string_copy(0x8000000000000003U, vek.wrapped_vek, 40, der, der_end);
// HW encrypt has more here ... TODO
return der != nullptr;
}
+68
查看文件
@@ -0,0 +1,68 @@
#pragma once
#include <cstdint>
#include <vector>
#include <string>
#include <iosfwd>
#include "Global.h"
class ApfsContainer;
struct bagdata_t;
struct key_header_t;
struct kek_entry_t;
struct vek_entry_t;
class Keybag
{
public:
Keybag();
~Keybag();
bool Init(const media_keybag_t *mk, size_t size);
size_t GetKeyCnt();
const keybag_entry_t * GetKey(size_t nr);
const keybag_entry_t * FindKey(const apfs_uuid_t &uuid, uint16_t type);
void dump(std::ostream &st, Keybag *cbag, const apfs_uuid_t &vuuid);
private:
std::vector<uint8_t> m_data;
kb_locker_t *m_kl;
};
class KeyManager
{
friend class Keybag;
public:
KeyManager(ApfsContainer &container);
~KeyManager();
bool Init(uint64_t block, uint64_t blockcnt, const apfs_uuid_t &container_uuid);
bool GetPasswordHint(std::string &hint, const apfs_uuid_t &volume_uuid);
bool GetVolumeKey(uint8_t *vek, const apfs_uuid_t &volume_uuid, const char *password = nullptr);
bool IsValid() const { return m_is_valid; }
bool IsUnencrypted() const { return m_is_unencrypted; }
void dump(std::ostream &st);
private:
bool LoadKeybag(Keybag &bag, uint32_t type, uint64_t block, uint64_t blockcnt, const apfs_uuid_t &uuid);
void DecryptBlocks(uint8_t *data, uint64_t block, uint64_t cnt, const uint8_t *key);
static const uint8_t* DecodeKeyHeader(key_header_t &hdr, const uint8_t*& end, const uint8_t* der, const uint8_t* der_end);
static bool DecodeKEK(kek_entry_t &kek, const uint8_t* der, const uint8_t* der_end);
static bool DecodeVEK(vek_entry_t &vek, const uint8_t* der, const uint8_t* der_end);
ApfsContainer &m_container;
Keybag m_container_bag;
apfs_uuid_t m_container_uuid;
bool m_is_valid;
bool m_is_unencrypted;
};
+476
查看文件
@@ -0,0 +1,476 @@
#include <string>
#include "PList.h"
PLObject::PLObject()
{
}
PLObject::~PLObject()
{
}
const PLInteger * PLObject::toInt() const
{
return dynamic_cast<const PLInteger *>(this);
}
const PLString * PLObject::toString() const
{
return dynamic_cast<const PLString *>(this);
}
const PLData * PLObject::toData() const
{
return dynamic_cast<const PLData *>(this);
}
const PLArray * PLObject::toArray() const
{
return dynamic_cast<const PLArray *>(this);
}
const PLDict * PLObject::toDict() const
{
return dynamic_cast<const PLDict *>(this);
}
PLInteger::PLInteger()
{
}
PLInteger::~PLInteger()
{
}
PLType PLInteger::type() const
{
return PLType_Integer;
}
PLString::PLString()
{
}
PLString::~PLString()
{
}
PLType PLString::type() const
{
return PLType_String;
}
PLData::PLData()
{
}
PLData::~PLData()
{
}
PLType PLData::type() const
{
return PLType_Data;
}
PLArray::PLArray()
{
}
PLArray::~PLArray()
{
for (std::vector<PLObject *>::iterator it = m_array.begin(); it != m_array.end(); it++)
delete *it;
m_array.clear();
}
PLType PLArray::type() const
{
return PLType_Array;
}
PLObject * PLArray::get(size_t idx) const
{
if (idx < m_array.size())
return m_array[idx];
else
return nullptr;
}
PLDict::PLDict()
{
}
PLDict::~PLDict()
{
for (std::map<std::string, PLObject*>::iterator it = m_dict.begin(); it != m_dict.end(); it++)
delete it->second;
m_dict.clear();
}
PLType PLDict::type() const
{
return PLType_Dict;
}
PLObject * PLDict::get(const char * name) const
{
std::map<std::string, PLObject *>::const_iterator it = m_dict.find(name);
if (it != m_dict.cend())
return it->second;
else
return nullptr;
}
PListXmlParser::PListXmlParser(const char * data, size_t size)
:
m_data(data),
m_size(size)
{
m_idx = 0;
}
PListXmlParser::~PListXmlParser()
{
}
PLObject * PListXmlParser::Parse()
{
std::string name;
std::string content;
TagType type;
PLObject *root = nullptr;
while (FindTag(name, type))
{
if (type == TagType::Start && name == "plist")
{
try
{
root = ParseObject();
}
catch (PLException &ex)
{
delete root;
root = nullptr;
fprintf(stderr, "XML PList parse error: %s\n", ex.what());
}
break;
}
}
return root;
}
PLArray * PListXmlParser::ParseArray()
{
PLArray *arr = new PLArray();
PLObject *obj;
for (;;)
{
obj = ParseObject();
if (obj)
arr->m_array.push_back(obj);
else
break;
}
return arr;
}
PLDict * PListXmlParser::ParseDict()
{
PLDict *dict = new PLDict();
PLObject *obj;
TagType tagtype;
std::string tagname;
std::string key;
for (;;)
{
FindTag(tagname, tagtype);
if (tagname == "dict" && tagtype == TagType::End)
break;
if (tagname != "key" || tagtype != TagType::Start)
throw PLException("Invalid tag in dict");
GetContent(key);
if (key.empty())
throw PLException("Empty key in dict");
FindTag(tagname, tagtype);
if (tagname != "key" || tagtype != TagType::End)
throw PLException("Invalid tag type, expected </key>");
obj = ParseObject();
if (obj)
dict->m_dict[key] = obj;
}
return dict;
}
PLObject * PListXmlParser::ParseObject()
{
std::string name;
TagType type;
std::string content_str;
const char *content_start;
size_t content_size;
PLObject *robj = nullptr;
FindTag(name, type);
if (type == TagType::Start)
{
if (name == "integer")
{
GetContent(content_str);
FindTag(name, type);
if (name != "integer" || type != TagType::End)
throw PLException("Invalid end tag, expected </integer>.");
PLInteger *obj = new PLInteger();
obj->m_value = strtoll(content_str.c_str(), nullptr, 0);
robj = obj;
}
else if (name == "string")
{
GetContent(content_str);
FindTag(name, type);
if (name != "string" || type != TagType::End)
throw PLException("Invalid end tag, expected </string>.");
PLString *obj = new PLString();
obj->m_string = content_str;
robj = obj;
}
else if (name == "data")
{
content_start = m_data + m_idx;
content_size = GetContentSize();
FindTag(name, type);
if (name != "data" || type != TagType::End)
throw PLException("Invalid end tag, expected </data>.");
PLData *obj = new PLData();
Base64Decode(obj->m_data, content_start, content_size);
robj = obj;
}
else if (name == "array")
{
PLArray *obj = ParseArray();
if (!obj)
return nullptr;
robj = obj;
}
else if (name == "dict")
{
PLDict *obj = ParseDict();
if (!obj)
return nullptr;
robj = obj;
}
else
{
throw PLException("Unexpected start tag.");
}
}
else if (type == TagType::Empty)
{
if (name == "true")
{
PLInteger *obj = new PLInteger();
obj->m_value = 1;
robj = obj;
}
else if (name == "false")
{
PLInteger *obj = new PLInteger();
obj->m_value = 0;
robj = obj;
}
else
{
throw PLException("Unexpected empty tag.");
}
}
else if (type == TagType::End)
{
return nullptr;
}
return robj;
}
void PListXmlParser::Base64Decode(std::vector<uint8_t>& bin, const char * str, size_t size)
{
int chcnt;
uint32_t buf;
size_t ip;
char ch;
uint32_t dec;
bin.clear();
bin.reserve(size * 4 / 3);
chcnt = 0;
buf = 0;
for (ip = 0; ip < size; ip++)
{
ch = str[ip];
if (ch >= 'A' && ch <= 'Z')
dec = ch - 'A';
else if (ch >= 'a' && ch <= 'z')
dec = ch - 'a' + 0x1A;
else if (ch >= '0' && ch <= '9')
dec = ch - '0' + 0x34;
else if (ch == '+')
dec = 0x3E;
else if (ch == '/')
dec = 0x3F;
else if (ch != '=')
continue;
else
break;
buf = (buf << 6) | dec;
chcnt++;
if (chcnt == 2)
bin.push_back((buf >> 4) & 0xFF);
else if (chcnt == 3)
bin.push_back((buf >> 2) & 0xFF);
else if (chcnt == 4)
{
bin.push_back((buf >> 0) & 0xFF);
chcnt = 0;
}
}
}
bool PListXmlParser::FindTag(std::string & name, TagType & type)
{
char ch;
bool in_name = false;
name.clear();
type = TagType::None;
do {
ch = GetChar();
} while (ch != '<' && ch != 0);
if (ch == 0)
return false;
ch = GetChar();
switch (ch)
{
case '?':
type = TagType::ProcInstr;
break;
case '!':
type = TagType::Doctype;
break;
case '/':
type = TagType::End;
in_name = true;
break;
default:
type = TagType::Start;
name.push_back(ch);
in_name = true;
break;
}
do
{
ch = GetChar();
switch (ch)
{
case 0x09:
case 0x0A:
case 0x0D:
case ' ':
case '>':
in_name = false;
break;
case '/':
if (m_data[m_idx] == '>')
type = TagType::Empty;
in_name = false;
break;
}
if (in_name)
name.push_back(ch);
} while (ch != '>' && ch != 0);
return true;
}
bool PListXmlParser::GetContent(std::string & content)
{
size_t start = m_idx;
content.clear();
while (m_idx < m_size && m_data[m_idx] != '<')
m_idx++;
if (m_idx == m_size)
return false;
content.assign(m_data + start, m_idx - start);
return true;
}
size_t PListXmlParser::GetContentSize()
{
size_t start = m_idx;
while (m_idx < m_size && m_data[m_idx] != '<')
m_idx++;
if (m_idx == m_size)
return m_idx - start;
return m_idx - start;
}
PList::PList()
{
m_root = nullptr;
}
PList::~PList()
{
}
bool PList::parseXML(const char * data, size_t size)
{
(void)data;
(void)size;
return false;
}
+182
查看文件
@@ -0,0 +1,182 @@
#pragma once
#include <vector>
#include <map>
#include <memory>
enum PLType
{
PLType_Integer,
PLType_String,
PLType_Data,
PLType_Array,
PLType_Dict
};
class PLInteger;
class PLString;
class PLData;
class PLArray;
class PLDict;
class PLException : public std::exception
{
public:
PLException(const char *reason) { m_reason = reason; }
const char *what() const noexcept override { return m_reason; }
private:
const char *m_reason;
};
class PLObject
{
protected:
PLObject();
public:
virtual ~PLObject();
virtual PLType type() const = 0;
const PLInteger *toInt() const;
const PLString *toString() const;
const PLData *toData() const;
const PLArray *toArray() const;
const PLDict *toDict() const;
};
class PLInteger : public PLObject
{
friend class PListXmlParser;
public:
PLInteger();
virtual ~PLInteger();
PLType type() const override;
int64_t value() const { return m_value; }
private:
int64_t m_value;
};
class PLString : public PLObject
{
friend class PListXmlParser;
public:
PLString();
virtual ~PLString();
PLType type() const override;
const std::string &string() const { return m_string; }
private:
std::string m_string;
};
class PLData : public PLObject
{
friend class PListXmlParser;
public:
PLData();
virtual ~PLData();
PLType type() const override;
const uint8_t *data() const { return m_data.data(); }
size_t size() const { return m_data.size(); }
private:
std::vector<uint8_t> m_data;
};
class PLArray : public PLObject
{
friend class PListXmlParser;
public:
PLArray();
virtual ~PLArray();
PLType type() const override;
PLObject *get(size_t idx) const;
size_t size() const { return m_array.size(); }
const std::vector<PLObject *> &array() const { return m_array; }
private:
std::vector<PLObject *> m_array;
};
class PLDict : public PLObject
{
friend class PListXmlParser;
public:
PLDict();
virtual ~PLDict();
PLType type() const override;
PLObject *get(const char *name) const;
const std::map<std::string, PLObject *> &dict() const { return m_dict; }
private:
std::map<std::string, PLObject *> m_dict;
};
class PListXmlParser
{
enum class TagType
{
None,
Empty,
Start,
End,
ProcInstr,
Doctype
};
public:
PListXmlParser(const char *data, size_t size);
~PListXmlParser();
PLObject *Parse();
private:
PLArray * ParseArray();
PLDict * ParseDict();
PLObject * ParseObject();
void Base64Decode(std::vector<uint8_t> &bin, const char *str, size_t size);
bool FindTag(std::string &name, TagType &type);
bool GetContent(std::string &content);
size_t GetContentSize();
char GetChar()
{
if (m_idx < m_size)
return m_data[m_idx++];
else
return 0;
}
const char * const m_data;
const size_t m_size;
size_t m_idx;
};
// Not used ... maybe later, if we need bplists ...
class PList
{
public:
PList();
~PList();
bool parseXML(const char *data, size_t size);
private:
PLObject * m_root;
};
+328
查看文件
@@ -0,0 +1,328 @@
#include "Unicode.h"
#include <cstdint>
#include <cstddef>
#include "UnicodeTables_v10.h"
int normalizeJimdo(char32_t ch, char32_t *nfd, uint8_t *ccc)
{
constexpr int SBase = 0xAC00;
constexpr int LBase = 0x1100;
constexpr int VBase = 0x1161;
constexpr int TBase = 0x11A7;
// constexpr int LCount = 19;
constexpr int VCount = 21;
constexpr int TCount = 28;
constexpr int NCount = VCount * TCount;
// constexpr int SCount = LCount * NCount;
int SIndex = ch - SBase;
int LIndex = SIndex / NCount;
int VIndex = (SIndex % NCount) / TCount;
int TIndex = SIndex % TCount;
nfd[0] = LBase + LIndex;
ccc[0] = 0;
nfd[1] = VBase + VIndex;
ccc[1] = 0;
if (TIndex > 0)
{
nfd[2] = TBase + TIndex;
ccc[2] = 0;
return 3;
}
return 2;
}
int normalizeOptFoldU32Char(char32_t ch, bool case_fold, char32_t *nfd, uint8_t *ccc)
{
char32_t ch_idx;
uint16_t hi_res;
uint16_t mi_res;
uint16_t lo_res;
const uint16_t *seq_u16 = 0;
const uint32_t *seq_u32 = 0;
uint32_t seq_len = 0;
uint32_t cnt;
char32_t c;
ccc[0] = 0;
if (ch >= 0xF0000)
{
if ((ch & 0xFFFE) == 0xFFFE)
return -1;
else
{
nfd[0] = ch;
return 1;
}
}
if (ch < 0x2FB00)
ch_idx = ch;
else if ((ch & 0xFFFFFE00) == 0xE0000)
ch_idx = ch - 0xB0500;
else
return -1;
hi_res = nf_trie_hi[ch_idx >> 8];
if (hi_res == 0xFFFF)
return -1;
if (hi_res == 0 || ((hi_res & 0xFF00) == 0xAD00))
{
nfd[0] = ch;
ccc[0] = hi_res & 0xFF;
return 1;
}
if (hi_res == 0xAC00) // Naja, fast ... sollte funktionieren
return normalizeJimdo(ch, nfd, ccc);
mi_res = nf_trie_mid[((hi_res & 0xFFF) << 4) | ((ch_idx >> 4) & 0xF)];
if (mi_res == 0xFFFF)
return -1;
if (mi_res == 0xAC00)
return normalizeJimdo(ch, nfd, ccc);
if (mi_res == 0 || (mi_res & 0xFF00) == 0xAD00)
{
ccc[0] = mi_res & 0xFF;
if (case_fold && (ch < 0x500))
nfd[0] = nf_basic_cf[ch];
else
nfd[0] = ch;
return 1;
}
if ((mi_res & 0xFF00) == 0xAE00)
{
uint16_t mask = nf_u16_inv_masks[mi_res & 0xFF];
if ((mask >> (ch_idx & 0xF)) & 1)
return -1;
if (case_fold && (ch < 0x500))
nfd[0] = nf_basic_cf[ch];
else
nfd[0] = ch;
return 1;
}
lo_res = nf_trie_lo[((mi_res & 0xFFF) << 4) | (ch_idx & 0xF)];
if (lo_res == 0xFFFF)
return -1;
if (lo_res == 0xAC00)
return normalizeJimdo(ch, nfd, ccc);
if (lo_res < 0xB000 || lo_res >= 0xF900)
{
if (lo_res == 0 || ((lo_res & 0xFF00) == 0xAD00))
ccc[0] = lo_res & 0xFF;
else
ch = lo_res;
if (case_fold && (ch < 0x500))
nfd[0] = nf_basic_cf[ch];
else
nfd[0] = ch;
return 1;
}
switch ((lo_res >> 12) & 0xF)
{
case 0xB:
if ((lo_res & 0x800) && !case_fold)
{
nfd[0] = ch;
return 1;
}
seq_u16 = nf_u16_seq_2 + 2 * (lo_res & 0x7FF);
seq_len = 2;
break;
case 0xC:
if ((lo_res & 0x800) && !case_fold)
{
nfd[0] = ch;
return 1;
}
seq_u16 = nf_u16_seq_3 + 3 * (lo_res & 0x7FF);
seq_len = 3;
break;
case 0xD:
seq_u16 = nf_u16_seq_misc + (lo_res & 0x3FF) + 1;
seq_len = nf_u16_seq_misc[lo_res & 0x3FF]; // Rest >> 4 in eax
ccc[0] = seq_len >> 4;
seq_len &= 0xF;
if (seq_len > 4)
return 0;
break;
case 0xE:
if ((lo_res & 0x800) && !case_fold)
{
nfd[0] = ch;
return 1;
}
seq_u32 = nf_u32_char + (lo_res & 0x7FF);
seq_len = 1;
break;
case 0xF:
seq_u32 = nf_u32_seq_misc + (lo_res & 0x3FF) + 1;
seq_len = nf_u32_seq_misc[lo_res & 0x3FF];
ccc[0] = seq_len >> 4;
seq_len &= 0xF;
if (seq_len > 4)
return 0;
break;
}
for (cnt = 0; cnt < seq_len; cnt++)
{
if (seq_u16)
c = seq_u16[cnt];
else
c = seq_u32[cnt];
nfd[cnt] = c;
if (cnt > 0)
{
if (c >= 0xF0000)
{
ccc[cnt] = 0;
continue;
}
if (c == 0x3B9)
{
ccc[cnt] = 0xF0;
continue;
}
ch_idx = (c > 0x2FB00) ? (c - 0xB0500) : c;
hi_res = nf_trie_hi[ch_idx >> 8];
if (hi_res == 0 || ((hi_res & 0xFF00) == 0xAD00))
{
ccc[cnt] = hi_res & 0xFF;
continue;
}
mi_res = nf_trie_mid[((hi_res & 0xFFF) << 4) | ((ch_idx >> 4) & 0xF)];
if (mi_res == 0 || ((mi_res & 0xFF00) == 0xAE00))
{
ccc[cnt] = 0;
continue;
}
if ((mi_res & 0xFF00) == 0xAD00)
{
ccc[cnt] = mi_res & 0xFF;
continue;
}
lo_res = nf_trie_lo[((mi_res & 0xFFF) << 4) | (ch_idx & 0xF)];
if ((lo_res & 0xFF00) == 0xAD00)
ccc[cnt] = lo_res & 0xFF;
else
ccc[cnt] = 0;
}
}
if (case_fold)
{
if (nfd[0] < 0x500)
nfd[0] = nf_basic_cf[nfd[0]];
if (cnt >= 2)
{
if (nfd[cnt - 1] == 0x345)
nfd[cnt - 1] = 0x3B9;
}
}
return cnt;
}
void CanonicalReorder(char32_t *nfd, uint8_t *ccc, size_t len)
{
size_t i;
size_t k;
bool sorted = false;
uint8_t sort_ccc;
char32_t sort_ch;
for (k = 0; k < (len - 1); k++)
{
if (ccc[k] == 0)
continue;
if (ccc[k + 1] == 0)
continue;
do
{
sorted = true;
for (i = k; i < (len - 1) && ccc[i + 1] != 0; i++)
{
if (ccc[i] > ccc[i + 1])
{
sorted = false;
sort_ccc = ccc[i + 1];
ccc[i + 1] = ccc[i];
ccc[i] = sort_ccc;
sort_ch = nfd[i + 1];
nfd[i + 1] = nfd[i];
nfd[i] = sort_ch;
}
}
} while (!sorted);
k = i + 1;
}
}
bool NormalizeFoldString(std::vector<char32_t> &nfd, const std::vector<char32_t> &in, bool case_fold)
{
char32_t ch;
size_t k;
size_t out_size;
int rc;
std::vector<uint8_t> ccc;
nfd.clear();
nfd.resize(in.size() * 4);
ccc.resize(nfd.size());
for (k = 0, out_size = 0; k < in.size(); k++)
{
ch = in[k];
rc = normalizeOptFoldU32Char(ch, case_fold, nfd.data() + out_size, ccc.data() + out_size);
if (rc == -1)
return false;
out_size += rc;
}
nfd.resize(out_size);
ccc.resize(out_size);
CanonicalReorder(nfd.data(), ccc.data(), nfd.size());
return true;
}
+10
查看文件
@@ -0,0 +1,10 @@
#pragma once
#include <cstdint>
#include <cstddef>
#include <vector>
int normalizeOptFoldU32Char(char32_t ch, bool case_insensitive, char32_t *sequence_out, unsigned char *unknown_out);
void CanonicalReorder(char32_t *nfd, uint8_t *ccc, size_t len);
bool NormalizeFoldString(std::vector<char32_t> &out, const std::vector<char32_t> &in, bool case_fold);
+1027
查看文件
文件差异过大,未显示。 加载差异
+792
查看文件
@@ -0,0 +1,792 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Unicode.h"
#include <cassert>
#include <iostream>
#include <iomanip>
#include <vector>
#include <sstream>
#if defined(__linux__) || defined(__APPLE__)
#include <termios.h>
#endif
#include <stdio.h>
#include "Util.h"
#include "Crc32.h"
#include <zlib.h>
#include <bzlib.h>
extern "C" {
#include <lzfse.h>
#include <lzvn_decode_base.h>
}
static Crc32 g_crc(true, 0x1EDC6F41);
uint64_t Fletcher64(const uint32_t *data, size_t cnt, uint64_t init)
{
size_t k;
uint64_t sum1 = init & 0xFFFFFFFFU;
uint64_t sum2 = (init >> 32);
for (k = 0; k < cnt; k++)
{
sum1 = (sum1 + data[k]);
sum2 = (sum2 + sum1);
}
sum1 = sum1 % 0xFFFFFFFF;
sum2 = sum2 % 0xFFFFFFFF;
return (static_cast<uint64_t>(sum2) << 32) | static_cast<uint64_t>(sum1);
}
bool VerifyBlock(const void *block, size_t size)
{
uint64_t cs;
const uint32_t * const data = reinterpret_cast<const uint32_t *>(block);
size /= sizeof(uint32_t);
cs = *reinterpret_cast<const uint64_t *>(block);
if (cs == 0)
return false;
if (cs == 0xFFFFFFFFFFFFFFFFULL)
return false;
cs = Fletcher64(data + 2, size - 2, 0);
cs = Fletcher64(data, 2, cs);
return cs == 0;
}
bool IsZero(const uint8_t* data, size_t size)
{
for (size_t k = 0; k < size; k++)
{
if (data[k] != 0)
return false;
}
return true;
}
bool IsEmptyBlock(const void *data, size_t blksize)
{
blksize /= sizeof(uint64_t);
const uint64_t *qdata = reinterpret_cast<const uint64_t *>(data);
for (size_t k = 0; k < blksize; k++)
{
if (qdata[k] != 0)
return false;
}
return true;
}
void DumpHex(std::ostream &os, const uint8_t *data, size_t size, size_t lineSize)
{
using namespace std;
size_t i, j;
uint8_t b;
if (size == 0)
return;
os << hex << uppercase << setfill('0');
for (i = 0; i < size; i += lineSize)
{
os << setw(4) << i << ": ";
for (j = 0; (j < lineSize) && ((i + j) < size); j++)
os << setw(2) << static_cast<unsigned int>(data[i + j]) << ' ';
for (; j < lineSize; j++)
os << " ";
os << "- ";
for (j = 0; (j < lineSize) && ((i + j) < size); j++)
{
b = data[i + j];
if (b >= 0x20 && b < 0x7F)
os << b;
else
os << '.';
}
os << endl;
}
// os << dec;
}
// Like DumpHex, but prints a label.
void DumpBuffer(const uint8_t *data, size_t len, const char *label)
{
std::cout << "dumping " << label << std::endl;
DumpHex(std::cout, data, len);
}
std::string uuidstr(const apfs_uuid_t &uuid)
{
using namespace std;
stringstream st;
int k;
st << hex << uppercase << setfill('0');
for (k = 0; k < 4; k++)
st << setw(2) << static_cast<unsigned>(uuid[k]);
st << '-';
for (k = 4; k < 6; k++)
st << setw(2) << static_cast<unsigned>(uuid[k]);
st << '-';
for (k = 6; k < 8; k++)
st << setw(2) << static_cast<unsigned>(uuid[k]);
st << '-';
for (k = 8; k < 10; k++)
st << setw(2) << static_cast<unsigned>(uuid[k]);
st << '-';
for (k = 10; k < 16; k++)
st << setw(2) << static_cast<unsigned>(uuid[k]);
return st.str();
}
std::string hexstr(const uint8_t *data, size_t size)
{
using namespace std;
std::ostringstream st;
size_t k;
st << hex << uppercase << setfill('0');
for (k = 0; k < size; k++)
st << setw(2) << static_cast<unsigned int>(data[k]);
return st.str();
}
void dump_utf8(std::ostream &st, const uint8_t* str)
{
size_t ptr = 0;
uint8_t ch;
char32_t uch;
int cnt;
st << std::hex << std::uppercase;
while (str[ptr] != 0)
{
ch = str[ptr++];
if (ch < 0x80) {
uch = ch;
cnt = 0;
} else if (ch < 0xC0) {
break;
} else if (ch < 0xE0) {
uch = ch & 0x1F;
cnt = 1;
} else if (ch < 0xF0) {
uch = ch & 0x0F;
cnt = 2;
} else if (ch < 0xF8) {
uch = ch & 0x07;
cnt = 3;
} else
break;
for (; cnt > 0; --cnt)
{
ch = str[ptr++];
if ((ch & 0xC0) == 0x80)
uch = (uch << 6) | (ch & 0x3F);
else
break;
}
st << uch << ' ';
}
st << str << std::endl;
}
void dump_utf32(std::ostream &st, const char32_t *str, size_t size)
{
size_t k;
st << std::hex << std::uppercase;
for (k = 0; k < size; k++)
st << str[k] << ' ';
for (k = 0; k < size; k++)
if (str[k] >= 0x20 && str[k] < 0x7F)
st << static_cast<char>(str[k]);
else
st << '.';
st << std::endl;
}
uint32_t HashFilename(const uint8_t* utf8str, uint16_t name_len, bool case_fold)
{
std::vector<char32_t> utf32;
std::vector<char32_t> utf32_nfd;
uint32_t hash;
Utf8toUtf32(utf32, utf8str);
NormalizeFoldString(utf32_nfd, utf32, case_fold);
#if 0
if (g_debug & Dbg_Dir)
{
dump_utf32(utf32.data(), utf32.size());
dump_utf32(utf32_nfd.data(), utf32_nfd.size());
}
#endif
g_crc.SetCRC(0xFFFFFFFF);
g_crc.Calc(reinterpret_cast<const uint8_t *>(utf32_nfd.data()), utf32_nfd.size() * sizeof(char32_t));
hash = g_crc.GetCRC();
hash = ((hash & 0x3FFFFF) << 10) | (name_len & 0x3FF);
return hash;
}
int apfs_strncmp(const uint8_t *s1, size_t s1_len, const uint8_t *s2, size_t s2_len)
{
while (s1_len > 0 && s2_len > 0) {
if (*s1 < *s2) return -1;
if (*s1 > *s2) return 1;
if (*s1 == 0) return 0;
++s1;
++s2;
--s1_len;
--s2_len;
}
if (s2_len) return -1;
if (s1_len) return 1;
return 0;
}
int StrCmpUtf8NormalizedFolded(const uint8_t* s1, const uint8_t* s2, bool case_fold)
{
std::vector<char32_t> s1_u32;
std::vector<char32_t> s2_u32;
std::vector<char32_t> s1_u32_nfd;
std::vector<char32_t> s2_u32_nfd;
size_t k;
Utf8toUtf32(s1_u32, s1);
Utf8toUtf32(s2_u32, s2);
NormalizeFoldString(s1_u32_nfd, s1_u32, case_fold);
NormalizeFoldString(s2_u32_nfd, s2_u32, case_fold);
s1_u32_nfd.push_back(0);
s2_u32_nfd.push_back(0);
k = 0;
while (true)
{
if (s1_u32_nfd[k] < s2_u32_nfd[k])
return -1;
if (s1_u32_nfd[k] > s2_u32_nfd[k])
return 1;
if (s1_u32_nfd[k] == 0 || s2_u32_nfd[k] == 0)
break;
++k;
}
return 0;
}
bool Utf8toUtf32(std::vector<char32_t> &str32, const uint8_t* str)
{
size_t ip = 0;
int cnt = 0;
char32_t ch;
uint8_t c;
bool ok = true;
while (str[ip] != 0)
{
c = str[ip++];
if (c < 0x80)
{
ch = c;
cnt = 0;
}
else if (c < 0xC0)
{
ok = false;
break;
}
else if (c < 0xE0)
{
ch = c & 0x1F;
cnt = 1;
}
else if (c < 0xF0)
{
ch = c & 0x0F;
cnt = 2;
}
else if (c < 0xF8)
{
ch = c & 0x07;
cnt = 3;
}
else
{
ok = false;
break;
}
for (; cnt > 0; --cnt)
{
c = str[ip++];
if ((c & 0xC0) == 0x80)
ch = (ch << 6) | (c & 0x3F);
else
{
ok = false;
break;
}
}
if (!ok)
break;
str32.push_back(ch);
if (ch == 0)
break;
}
return ok;
}
bool GetPassword(std::string &pw)
{
#if defined(__linux__) || defined(__APPLE__)
struct termios told, tnew;
FILE *stream = stdin;
/* Turn echoing off and fail if we can’t. */
if (tcgetattr (fileno (stream), &told) != 0)
return false;
tnew = told;
tnew.c_lflag &= ~ECHO;
if (tcsetattr (fileno (stream), TCSAFLUSH, &tnew) != 0)
return false;
/* Read the password. */
std::getline(std::cin, pw);
/* Restore terminal. */
(void) tcsetattr (fileno (stream), TCSAFLUSH, &told);
std::cout << std::endl;
return true;
#else
std::getline(std::cin, pw);
std::cout << std::endl;
return true;
#endif
}
size_t DecompressZLib(uint8_t *dst, size_t dst_size, const uint8_t *src, size_t src_size)
{
// size_t nwr = 0;
int ret;
z_stream strm;
memset(&strm, 0, sizeof(strm));
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.avail_in = src_size;
strm.avail_out = dst_size;
strm.next_in = const_cast<uint8_t *>(src);
strm.next_out = dst;
ret = inflateInit2(&strm, 15);
if (ret != Z_OK)
{
std::cerr << "DecompressZLib: inflateInit failed." << std::endl;
return 0;
}
do {
ret = inflate(&strm, Z_NO_FLUSH);
if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR)
{
strm.avail_out = 0;
break;
}
} while (ret != Z_STREAM_END);
inflateEnd(&strm);
return dst_size - strm.avail_out;
}
size_t DecompressADC(uint8_t * dst, size_t dst_size, const uint8_t * src, size_t src_size)
{
size_t in_idx = 0;
size_t out_idx = 0;
uint8_t ctl;
int len;
int dist;
int cnt;
for (;;)
{
if (in_idx >= src_size)
break;
if (out_idx >= dst_size)
break;
ctl = src[in_idx++];
if (ctl & 0x80)
{
len = (ctl & 0x7F) + 1;
for (cnt = 0; cnt < len; cnt++)
dst[out_idx++] = src[in_idx++];
}
else
{
if (ctl & 0x40)
{
len = ctl - 0x3C;
dist = src[in_idx++];
dist = (dist << 8) | src[in_idx++];
dist += 1;
}
else
{
len = ((ctl >> 2) & 0xF) + 3;
dist = (((ctl & 3) << 8) | src[in_idx++]) + 1;
}
for (cnt = 0; cnt < len; cnt++)
{
dst[out_idx] = dst[out_idx - dist];
out_idx++;
}
}
}
assert(in_idx == src_size);
assert(out_idx == dst_size);
return out_idx;
}
size_t DecompressLZVN(uint8_t * dst, size_t dst_size, const uint8_t * src, size_t src_size)
{
lzvn_decoder_state state;
memset(&state, 0, sizeof(state));
state.src = src;
state.src_end = src + src_size;
state.dst = dst;
state.dst_begin = dst;
state.dst_end = dst + dst_size;
state.dst_current = dst;
lzvn_decode(&state);
return static_cast<size_t>(state.dst - dst);
}
size_t DecompressBZ2(uint8_t * dst, size_t dst_size, const uint8_t * src, size_t src_size)
{
bz_stream strm;
memset(&strm, 0, sizeof(strm));
BZ2_bzDecompressInit(&strm, 0, 0);
strm.next_in = const_cast<char *>(reinterpret_cast<const char *>(src));
strm.avail_in = src_size;
strm.next_out = reinterpret_cast<char *>(dst);
strm.avail_out = dst_size;
BZ2_bzDecompress(&strm);
BZ2_bzDecompressEnd(&strm);
return strm.total_out_lo32;
}
size_t DecompressLZFSE(uint8_t * dst, size_t dst_size, const uint8_t * src, size_t src_size)
{
return lzfse_decode_buffer(dst, dst_size, src, src_size, nullptr);
}
size_t DecompressLZBITMAP(uint8_t* dst, size_t dst_size, const uint8_t* src, size_t src_size)
{
uint8_t scratch[0x2000];
uint8_t flags;
uint32_t compressed_size;
uint32_t uncompressed_size;
uint32_t distances_offset;
uint32_t bitmap_offset;
uint32_t token_offset;
uint8_t token_map_bmp[16] = {};
uint8_t token_map_tkn[16] = {};
const uint8_t* inp = src;
const uint8_t* in_end = src + src_size;
uint8_t* outp = dst;
const uint8_t* out_end = dst + dst_size;
const uint8_t* block_end;
const uint8_t* bitmaps;
const uint8_t* literals;
const uint8_t* distances;
unsigned n;
unsigned m;
unsigned o;
const uint8_t* p;
uint8_t tkn_split = 3; // TODO is dependent on flag ...
uint8_t t;
uint32_t rlecnt;
uint16_t bitbuf;
int nbits;
uint32_t ntoken;
uint32_t tmp_size;
uint8_t bmp;
uint8_t tkn;
uint32_t dist;
uint32_t bp;
uint32_t dp;
uint32_t lp;
if (inp[0] != 'Z' || inp[1] != 'B' || inp[2] != 'M' || (inp[3] & 0xF0) != 0)
return 0;
flags = inp[3];
assert(flags == 9);
if (flags != 9) {
log_error("LZBITMAP: Flags != 0x09\n");
return 0;
}
inp += 4;
while (true) {
compressed_size = inp[0] | inp[1] << 8 | inp[2] << 16;
uncompressed_size = inp[3] | inp[4] << 8 | inp[5] << 16;
if (compressed_size == 6) {
if (uncompressed_size != 0)
log_error("LZBITMAP: Compressed size = 6, uncompressed size != 0???\n");
break;
}
if (compressed_size == uncompressed_size + 6) {
if (outp + uncompressed_size > out_end || inp + compressed_size > in_end) {
log_error("LZBITMAP: Buffer overrun.\n");
break;
}
memcpy(outp, inp + 6, uncompressed_size);
outp += uncompressed_size;
inp += compressed_size;
continue;
}
distances_offset = inp[6] | inp[7] << 8 | inp[8] << 16;
bitmap_offset = inp[9] | inp[10] << 8 | inp[11] << 16;
token_offset = inp[12] | inp[13] << 8 | inp[14] << 16;
literals = inp + 15;
distances = inp + distances_offset;
bitmaps = inp + bitmap_offset;
p = inp + compressed_size - 0x11;
for (n = 0; n < tkn_split; n++) {
token_map_bmp[n] = 0;
token_map_tkn[n] = n;
}
nbits = 0;
bitbuf = 0;
for (; n < 15; n++) {
while (nbits < 10) {
bitbuf |= *p << nbits;
nbits += 8;
p++;
}
token_map_bmp[n] = bitbuf & 0xFF;
token_map_tkn[n] = bitbuf >> 8 & 3;
bitbuf >>= 10;
nbits -= 10;
}
ntoken = (uncompressed_size + 7) >> 3;
tmp_size = compressed_size - token_offset - 0x11;
for (n = 0; n < tmp_size; n++) {
scratch[2 * n + 0x1000] = inp[token_offset + n] & 0xF;
scratch[2 * n + 0x1001] = inp[token_offset + n] >> 4;
}
tmp_size *= 2;
p = scratch + 0x1000;
o = 0;
for (n = 0; n < tmp_size; n++) {
t = p[n];
if (t == 0xF) {
t = p[n - 1];
rlecnt = 3;
do {
n++;
rlecnt += p[n];
} while (p[n] == 0xF);
for (m = 0; m < rlecnt; m++)
scratch[o++] = t;
}
else {
scratch[o++] = t;
}
}
if (o != ntoken)
log_debug("RLE size = %X, ntoken = %X\n", o, ntoken);
o = 0;
m = 0;
dist = 8;
dp = 0;
bp = 0;
lp = 0;
block_end = outp + uncompressed_size;
assert(block_end <= out_end);
if (block_end > out_end) {
log_error("LZBITMAP: Output buffer too small.\n");
block_end = out_end;
}
for (n = 0; n < ntoken; n++) {
t = scratch[n];
if (t < tkn_split)
bmp = bitmaps[bp++];
else
bmp = token_map_bmp[t];
tkn = token_map_tkn[t];
if (tkn == 1) {
dist = distances[dp++];
}
else if (tkn == 2) {
dist = distances[dp] | distances[dp + 1] << 8;
dp += 2;
}
for (m = 0; m < 8 && outp < block_end; m++) {
if (bmp & 1)
*outp = literals[lp++];
else
*outp = *(outp - dist);
bmp >>= 1;
outp++;
}
}
if (lp != distances_offset - 15)
log_error("lp = %06X (%06X)\n", lp, distances_offset - 15);
if (dp != bitmap_offset - distances_offset)
log_error("dp = %06X (%06X)\n", dp, bitmap_offset - distances_offset);
if (bp != token_offset - bitmap_offset)
log_error("bp = %06X (%06X)\n", bp, token_offset - bitmap_offset);
inp += compressed_size;
}
return outp - dst;
}
int log2(uint32_t val)
{
int r = 0;
if (val & 0xFFFF0000) { val >>= 16; r += 16; }
if (val & 0xFF00) { val >>= 8; r += 8; }
if (val & 0xF0) { val >>= 4; r += 4; }
if (val & 0xC) { val >>= 2; r += 2; }
if (val & 0x2) { val >>= 1; r += 1; }
return r;
}
FILE *g_log = stderr;
void log_debug(const char *msg, ...)
{
if (g_debug & Dbg_Cmpfs) {
va_list va;
va_start(va, msg);
vfprintf(g_log, msg, va);
va_end(va);
}
}
void log_warn(const char *msg, ...)
{
if (g_debug & Dbg_Info) {
va_list va;
va_start(va, msg);
vfprintf(g_log, msg, va);
va_end(va);
}
}
void log_error(const char *msg, ...)
{
if (g_debug & Dbg_Errors) {
va_list va;
va_start(va, msg);
vfprintf(g_log, msg, va);
va_end(va);
}
}
+62
查看文件
@@ -0,0 +1,62 @@
/*
This file is part of apfs-fuse, a read-only implementation of APFS
(Apple File System) for FUSE.
Copyright (C) 2017 Simon Gander
Apfs-fuse is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Apfs-fuse is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with apfs-fuse. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <cstdint>
#include <cstddef>
#include <string>
#include <ostream>
#include <vector>
#include "ApfsTypes.h"
uint64_t Fletcher64(const uint32_t *data, size_t cnt, uint64_t init);
bool VerifyBlock(const void *block, size_t size);
bool IsZero(const uint8_t *data, size_t size);
bool IsEmptyBlock(const void *data, size_t blksize);
void DumpHex(std::ostream &os, const uint8_t *data, size_t size, size_t line_size = 16);
void DumpBuffer(const uint8_t *data, size_t len, const char *label);
std::string uuidstr(const apfs_uuid_t &uuid);
std::string hexstr(const uint8_t *data, size_t size);
void dump_utf8(std::ostream &st, const uint8_t *str);
void dump_utf32(std::ostream &st, const char32_t *str, size_t size);
uint32_t HashFilename(const uint8_t *utf8str, uint16_t name_len, bool case_fold);
int apfs_strncmp(const uint8_t *s1, size_t s1_len, const uint8_t *s2, size_t s2_len);
int StrCmpUtf8NormalizedFolded(const uint8_t *s1, const uint8_t *s2, bool case_fold);
bool Utf8toUtf32(std::vector<char32_t> &str32, const uint8_t * str);
size_t DecompressZLib(uint8_t *dst, size_t dst_size, const uint8_t *src, size_t src_size);
size_t DecompressADC(uint8_t *dst, size_t dst_size, const uint8_t *src, size_t src_size);
size_t DecompressLZVN(uint8_t *dst, size_t dst_size, const uint8_t *src, size_t src_size);
size_t DecompressBZ2(uint8_t *dst, size_t dst_size, const uint8_t *src, size_t src_size);
size_t DecompressLZFSE(uint8_t *dst, size_t dst_size, const uint8_t *src, size_t src_size);
size_t DecompressLZBITMAP(uint8_t *dst, size_t dst_size, const uint8_t *src, size_t src_size);
bool GetPassword(std::string &pw);
int log2(uint32_t val);
void log_debug(const char *msg, ...) __attribute__((format(printf, 1, 2)));
void log_warn(const char *msg, ...) __attribute__((format(printf, 1, 2)));
void log_error(const char *msg, ...) __attribute__((format(printf, 1, 2)));
+156
查看文件
@@ -0,0 +1,156 @@
#include <ApfsLib/ApfsContainer.h>
#include <ApfsLib/Device.h>
#include <ApfsLib/GptPartitionMap.h>
#include <cerrno>
#include <cinttypes>
static void print_apfs_uuid(const apfs_uuid_t &uuid)
{
printf("%02X%02X%02X%02X-", uuid[0], uuid[1], uuid[2], uuid[3]);
printf("%02X%02X-", uuid[4], uuid[5]);
printf("%02X%02X-", uuid[6], uuid[7]);
printf("%02X%02X-", uuid[8], uuid[9]);
printf("%02X%02X%02X%02X%02X%02X", uuid[10], uuid[11], uuid[12], uuid[13], uuid[14], uuid[15]);
}
static void print_role(uint16_t role)
{
static const char *rolestr[] = {
"System", "User", "Recovery", "VM", "Preboot", "Installer"
};
static const char *rolestr_enum[] = {
"", "Data", "Baseband", "Update",
"Xart", "Hardware", "Backup", "Reserved-7",
"Reserved-8", "Enterprise", "Reserved-10", "Prelogin"
};
bool first = true;
int k;
if (role == 0)
printf("No specific role");
else if (role <= 0x20) {
for (k = 0; k < 6; k++) {
if (role & (1 << k)) {
if (!first)
printf(", ");
printf("%s", rolestr[k]);
first = false;
}
}
} else if (role <= APFS_VOL_ROLE_PRELOGIN) {
printf("%s", rolestr_enum[role >> APFS_VOLUME_ENUM_SHIFT]);
}
// printf(" (%d)", role);
}
static void print_filevault(uint64_t flags)
{
if (flags == 1)
printf("No");
else
printf("Yes");
}
int main(int argc, char *argv[])
{
const char *devname = nullptr;
Device *device = nullptr;
ApfsContainer *container = nullptr;
uint64_t offset;
uint64_t size;
apfs_superblock_t apsb;
g_debug = 0;
if (argc < 2)
{
printf("Syntax: %s [device]\n", argv[0]);
return EINVAL;
}
devname = argv[1];
device = Device::OpenDevice(devname);
if (device) {
offset = 0;
size = device->GetSize();
GptPartitionMap gpt;
if (gpt.LoadAndVerify(*device)) {
printf("Found partitions:\n");
gpt.ListEntries();
int partnum = gpt.FindFirstAPFSPartition();
if (partnum >= 0) {
printf("First APFS partition is %d\n", partnum);
gpt.GetPartitionOffsetAndSize(partnum, offset, size);
}
printf("\n");
}
container = new ApfsContainer(device, offset, size);
if (container->Init()) {
// printf("Listing volumes:\n");
for (int k = 0; k < NX_MAX_FILE_SYSTEMS; k++) {
if (!container->GetVolumeInfo(k, apsb))
continue;
printf("Volume %d ", k);
print_apfs_uuid(apsb.apfs_vol_uuid);
printf("\n");
printf("---------------------------------------------\n");
printf("Role: ");
print_role(apsb.apfs_role);
printf("\n");
printf("Name: %s", apsb.apfs_volname);
if (apsb.apfs_incompatible_features & APFS_INCOMPAT_CASE_INSENSITIVE)
printf(" (Case-insensitive)\n");
else if (apsb.apfs_incompatible_features & APFS_INCOMPAT_NORMALIZATION_INSENSITIVE)
printf(" (Case-sensitive)\n");
else
printf("\n");
printf("Capacity Consumed: %" PRIu64 " Bytes\n", apsb.apfs_fs_alloc_count * container->GetBlocksize());
printf("FileVault: ");
print_filevault(apsb.apfs_fs_flags & APFS_FS_CRYPTOFLAGS);
printf("\n");
if (apsb.apfs_snap_meta_tree_oid) {
printf("Snapshots:\n");
BTree snap_tree(*container);
BTreeIterator it;
BTreeEntry bte;
const j_snap_metadata_key_t *sme_k;
const j_snap_metadata_val_t *sme_v;
snap_tree.Init(apsb.apfs_snap_meta_tree_oid, apsb.apfs_o.o_xid);
if (snap_tree.GetIteratorBegin(it)) {
for (;;) {
if (!it.GetEntry(bte)) break;
sme_k = reinterpret_cast<const j_snap_metadata_key_t*>(bte.key);
sme_v = reinterpret_cast<const j_snap_metadata_val_t*>(bte.val);
if ((sme_k->hdr.obj_id_and_type >> OBJ_TYPE_SHIFT) != APFS_TYPE_SNAP_METADATA) break;
printf(" %" PRIu64 " : '%s'\n", sme_k->hdr.obj_id_and_type & OBJ_ID_MASK, sme_v->name);
if (!it.next()) break;
}
}
}
printf("\n");
}
} else {
printf("Unable to open APFS container\n");
}
delete container;
device->Close();
delete device;
} else {
printf("Error opening device.\n");
return EIO;
}
return 0;
}
+142
查看文件
@@ -0,0 +1,142 @@
cmake_minimum_required(VERSION 3.0)
# set(CMAKE_C_COMPILER "clang")
# set(CMAKE_CXX_COMPILER "clang++")
project(Apfs)
option(USE_FUSE3 "Use the FUSE 3 library (required on 32-bit systems)" ON)
set(CMAKE_C_STANDARD 99)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_BUILD_TYPE Release)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra")
# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -march=native -fsanitize=undefined")
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -march=native -fsanitize=undefined")
# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
include_directories(. 3rdparty/lzfse/src)
add_library(lzfse
3rdparty/lzfse/src/lzfse.h
3rdparty/lzfse/src/lzfse_decode.c
3rdparty/lzfse/src/lzfse_decode_base.c
3rdparty/lzfse/src/lzfse_encode.c
3rdparty/lzfse/src/lzfse_encode_base.c
3rdparty/lzfse/src/lzfse_encode_tables.h
3rdparty/lzfse/src/lzfse_fse.c
3rdparty/lzfse/src/lzfse_fse.h
3rdparty/lzfse/src/lzfse_internal.h
3rdparty/lzfse/src/lzfse_tunables.h
3rdparty/lzfse/src/lzvn_decode_base.c
3rdparty/lzfse/src/lzvn_decode_base.h
3rdparty/lzfse/src/lzvn_encode_base.c
3rdparty/lzfse/src/lzvn_encode_base.h
)
add_library(crypto
Crypto/Aes.cpp
Crypto/Aes.h
Crypto/AesXts.cpp
Crypto/AesXts.h
Crypto/Asn1Der.cpp
Crypto/Asn1Der.h
Crypto/Crypto.cpp
Crypto/Crypto.h
Crypto/Des.cpp
Crypto/Des.h
Crypto/Sha1.cpp
Crypto/Sha1.h
Crypto/Sha256.cpp
Crypto/Sha256.h
Crypto/TripleDes.cpp
Crypto/TripleDes.h
)
add_library(apfs
ApfsLib/ApfsContainer.cpp
ApfsLib/ApfsContainer.h
ApfsLib/ApfsDir.cpp
ApfsLib/ApfsDir.h
ApfsLib/ApfsNodeMapper.cpp
ApfsLib/ApfsNodeMapper.h
ApfsLib/ApfsNodeMapperBTree.cpp
ApfsLib/ApfsNodeMapperBTree.h
ApfsLib/ApfsVolume.cpp
ApfsLib/ApfsVolume.h
ApfsLib/BlockDumper.cpp
ApfsLib/BlockDumper.h
ApfsLib/BTree.cpp
ApfsLib/BTree.h
ApfsLib/CheckPointMap.cpp
ApfsLib/CheckPointMap.h
ApfsLib/Crc32.cpp
ApfsLib/Crc32.h
ApfsLib/Decmpfs.cpp
ApfsLib/Decmpfs.h
ApfsLib/Device.cpp
ApfsLib/Device.h
ApfsLib/DeviceDMG.cpp
ApfsLib/DeviceDMG.h
ApfsLib/DeviceLinux.cpp
ApfsLib/DeviceLinux.h
ApfsLib/DeviceMac.cpp
ApfsLib/DeviceMac.h
ApfsLib/DeviceSparseImage.cpp
ApfsLib/DeviceSparseImage.h
ApfsLib/DeviceWinFile.cpp
ApfsLib/DeviceWinFile.h
ApfsLib/DeviceWinPhys.cpp
ApfsLib/DeviceWinPhys.h
ApfsLib/DiskImageFile.cpp
ApfsLib/DiskImageFile.h
ApfsLib/DiskStruct.h
ApfsLib/Endian.h
ApfsLib/Global.h
ApfsLib/GptPartitionMap.cpp
ApfsLib/GptPartitionMap.h
ApfsLib/KeyMgmt.cpp
ApfsLib/KeyMgmt.h
ApfsLib/PList.cpp
ApfsLib/PList.h
ApfsLib/Util.cpp
ApfsLib/Util.h
ApfsLib/Unicode.cpp
ApfsLib/Unicode.h)
target_link_libraries(apfs z bz2 lzfse crypto)
target_compile_definitions(apfs PUBLIC _FILE_OFFSET_BITS=64 _DARWIN_USE_64_BIT_INODE)
add_executable(apfs-dump
ApfsDump/Dumper.cpp
ApfsDump/Dumper.h
ApfsDump/Apfs.cpp)
target_link_libraries(apfs-dump apfs)
add_executable(apfs-dump-quick
ApfsDumpQuick/ApfsDumpQuick.cpp)
target_link_libraries(apfs-dump-quick apfs)
add_executable(apfs-fuse
apfsfuse/ApfsFuse.cpp)
target_compile_definitions(apfs-fuse PRIVATE _FILE_OFFSET_BITS=64 _DARWIN_USE_64_BIT_INODE)
if (APPLE)
target_include_directories(apfs-fuse PRIVATE /usr/local/include/osxfuse/)
# link_directories(/usr/local/lib/)
target_link_libraries(apfs-fuse apfs /usr/local/lib/libosxfuse.dylib)
else()
if (USE_FUSE3)
target_link_libraries(apfs-fuse apfs fuse3)
else()
target_link_libraries(apfs-fuse apfs fuse)
target_compile_definitions(apfs-fuse PRIVATE USE_FUSE2)
endif()
endif()
add_executable(apfsutil ApfsUtil/ApfsUtil.cpp)
target_link_libraries(apfsutil apfs)
include(GNUInstallDirs)
install(TARGETS apfs-fuse RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(TARGETS apfsutil RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
+584
查看文件
@@ -0,0 +1,584 @@
#include <stdint.h>
#include <stddef.h>
#include <algorithm>
// #include "Endian.h"
#include "Aes.h"
const uint32_t AES::Te0[256] = {
0xC66363A5U, 0xF87C7C84U, 0xEE777799U, 0xF67B7B8DU, 0xFFF2F20DU, 0xD66B6BBDU, 0xDE6F6FB1U, 0x91C5C554U,
0x60303050U, 0x02010103U, 0xCE6767A9U, 0x562B2B7DU, 0xE7FEFE19U, 0xB5D7D762U, 0x4DABABE6U, 0xEC76769AU,
0x8FCACA45U, 0x1F82829DU, 0x89C9C940U, 0xFA7D7D87U, 0xEFFAFA15U, 0xB25959EBU, 0x8E4747C9U, 0xFBF0F00BU,
0x41ADADECU, 0xB3D4D467U, 0x5FA2A2FDU, 0x45AFAFEAU, 0x239C9CBFU, 0x53A4A4F7U, 0xE4727296U, 0x9BC0C05BU,
0x75B7B7C2U, 0xE1FDFD1CU, 0x3D9393AEU, 0x4C26266AU, 0x6C36365AU, 0x7E3F3F41U, 0xF5F7F702U, 0x83CCCC4FU,
0x6834345CU, 0x51A5A5F4U, 0xD1E5E534U, 0xF9F1F108U, 0xE2717193U, 0xABD8D873U, 0x62313153U, 0x2A15153FU,
0x0804040CU, 0x95C7C752U, 0x46232365U, 0x9DC3C35EU, 0x30181828U, 0x379696A1U, 0x0A05050FU, 0x2F9A9AB5U,
0x0E070709U, 0x24121236U, 0x1B80809BU, 0xDFE2E23DU, 0xCDEBEB26U, 0x4E272769U, 0x7FB2B2CDU, 0xEA75759FU,
0x1209091BU, 0x1D83839EU, 0x582C2C74U, 0x341A1A2EU, 0x361B1B2DU, 0xDC6E6EB2U, 0xB45A5AEEU, 0x5BA0A0FBU,
0xA45252F6U, 0x763B3B4DU, 0xB7D6D661U, 0x7DB3B3CEU, 0x5229297BU, 0xDDE3E33EU, 0x5E2F2F71U, 0x13848497U,
0xA65353F5U, 0xB9D1D168U, 0x00000000U, 0xC1EDED2CU, 0x40202060U, 0xE3FCFC1FU, 0x79B1B1C8U, 0xB65B5BEDU,
0xD46A6ABEU, 0x8DCBCB46U, 0x67BEBED9U, 0x7239394BU, 0x944A4ADEU, 0x984C4CD4U, 0xB05858E8U, 0x85CFCF4AU,
0xBBD0D06BU, 0xC5EFEF2AU, 0x4FAAAAE5U, 0xEDFBFB16U, 0x864343C5U, 0x9A4D4DD7U, 0x66333355U, 0x11858594U,
0x8A4545CFU, 0xE9F9F910U, 0x04020206U, 0xFE7F7F81U, 0xA05050F0U, 0x783C3C44U, 0x259F9FBAU, 0x4BA8A8E3U,
0xA25151F3U, 0x5DA3A3FEU, 0x804040C0U, 0x058F8F8AU, 0x3F9292ADU, 0x219D9DBCU, 0x70383848U, 0xF1F5F504U,
0x63BCBCDFU, 0x77B6B6C1U, 0xAFDADA75U, 0x42212163U, 0x20101030U, 0xE5FFFF1AU, 0xFDF3F30EU, 0xBFD2D26DU,
0x81CDCD4CU, 0x180C0C14U, 0x26131335U, 0xC3ECEC2FU, 0xBE5F5FE1U, 0x359797A2U, 0x884444CCU, 0x2E171739U,
0x93C4C457U, 0x55A7A7F2U, 0xFC7E7E82U, 0x7A3D3D47U, 0xC86464ACU, 0xBA5D5DE7U, 0x3219192BU, 0xE6737395U,
0xC06060A0U, 0x19818198U, 0x9E4F4FD1U, 0xA3DCDC7FU, 0x44222266U, 0x542A2A7EU, 0x3B9090ABU, 0x0B888883U,
0x8C4646CAU, 0xC7EEEE29U, 0x6BB8B8D3U, 0x2814143CU, 0xA7DEDE79U, 0xBC5E5EE2U, 0x160B0B1DU, 0xADDBDB76U,
0xDBE0E03BU, 0x64323256U, 0x743A3A4EU, 0x140A0A1EU, 0x924949DBU, 0x0C06060AU, 0x4824246CU, 0xB85C5CE4U,
0x9FC2C25DU, 0xBDD3D36EU, 0x43ACACEFU, 0xC46262A6U, 0x399191A8U, 0x319595A4U, 0xD3E4E437U, 0xF279798BU,
0xD5E7E732U, 0x8BC8C843U, 0x6E373759U, 0xDA6D6DB7U, 0x018D8D8CU, 0xB1D5D564U, 0x9C4E4ED2U, 0x49A9A9E0U,
0xD86C6CB4U, 0xAC5656FAU, 0xF3F4F407U, 0xCFEAEA25U, 0xCA6565AFU, 0xF47A7A8EU, 0x47AEAEE9U, 0x10080818U,
0x6FBABAD5U, 0xF0787888U, 0x4A25256FU, 0x5C2E2E72U, 0x381C1C24U, 0x57A6A6F1U, 0x73B4B4C7U, 0x97C6C651U,
0xCBE8E823U, 0xA1DDDD7CU, 0xE874749CU, 0x3E1F1F21U, 0x964B4BDDU, 0x61BDBDDCU, 0x0D8B8B86U, 0x0F8A8A85U,
0xE0707090U, 0x7C3E3E42U, 0x71B5B5C4U, 0xCC6666AAU, 0x904848D8U, 0x06030305U, 0xF7F6F601U, 0x1C0E0E12U,
0xC26161A3U, 0x6A35355FU, 0xAE5757F9U, 0x69B9B9D0U, 0x17868691U, 0x99C1C158U, 0x3A1D1D27U, 0x279E9EB9U,
0xD9E1E138U, 0xEBF8F813U, 0x2B9898B3U, 0x22111133U, 0xD26969BBU, 0xA9D9D970U, 0x078E8E89U, 0x339494A7U,
0x2D9B9BB6U, 0x3C1E1E22U, 0x15878792U, 0xC9E9E920U, 0x87CECE49U, 0xAA5555FFU, 0x50282878U, 0xA5DFDF7AU,
0x038C8C8FU, 0x59A1A1F8U, 0x09898980U, 0x1A0D0D17U, 0x65BFBFDAU, 0xD7E6E631U, 0x844242C6U, 0xD06868B8U,
0x824141C3U, 0x299999B0U, 0x5A2D2D77U, 0x1E0F0F11U, 0x7BB0B0CBU, 0xA85454FCU, 0x6DBBBBD6U, 0x2C16163AU
};
const uint32_t AES::Te1[256] = {
0xA5C66363U, 0x84F87C7CU, 0x99EE7777U, 0x8DF67B7BU, 0x0DFFF2F2U, 0xBDD66B6BU, 0xB1DE6F6FU, 0x5491C5C5U,
0x50603030U, 0x03020101U, 0xA9CE6767U, 0x7D562B2BU, 0x19E7FEFEU, 0x62B5D7D7U, 0xE64DABABU, 0x9AEC7676U,
0x458FCACAU, 0x9D1F8282U, 0x4089C9C9U, 0x87FA7D7DU, 0x15EFFAFAU, 0xEBB25959U, 0xC98E4747U, 0x0BFBF0F0U,
0xEC41ADADU, 0x67B3D4D4U, 0xFD5FA2A2U, 0xEA45AFAFU, 0xBF239C9CU, 0xF753A4A4U, 0x96E47272U, 0x5B9BC0C0U,
0xC275B7B7U, 0x1CE1FDFDU, 0xAE3D9393U, 0x6A4C2626U, 0x5A6C3636U, 0x417E3F3FU, 0x02F5F7F7U, 0x4F83CCCCU,
0x5C683434U, 0xF451A5A5U, 0x34D1E5E5U, 0x08F9F1F1U, 0x93E27171U, 0x73ABD8D8U, 0x53623131U, 0x3F2A1515U,
0x0C080404U, 0x5295C7C7U, 0x65462323U, 0x5E9DC3C3U, 0x28301818U, 0xA1379696U, 0x0F0A0505U, 0xB52F9A9AU,
0x090E0707U, 0x36241212U, 0x9B1B8080U, 0x3DDFE2E2U, 0x26CDEBEBU, 0x694E2727U, 0xCD7FB2B2U, 0x9FEA7575U,
0x1B120909U, 0x9E1D8383U, 0x74582C2CU, 0x2E341A1AU, 0x2D361B1BU, 0xB2DC6E6EU, 0xEEB45A5AU, 0xFB5BA0A0U,
0xF6A45252U, 0x4D763B3BU, 0x61B7D6D6U, 0xCE7DB3B3U, 0x7B522929U, 0x3EDDE3E3U, 0x715E2F2FU, 0x97138484U,
0xF5A65353U, 0x68B9D1D1U, 0x00000000U, 0x2CC1EDEDU, 0x60402020U, 0x1FE3FCFCU, 0xC879B1B1U, 0xEDB65B5BU,
0xBED46A6AU, 0x468DCBCBU, 0xD967BEBEU, 0x4B723939U, 0xDE944A4AU, 0xD4984C4CU, 0xE8B05858U, 0x4A85CFCFU,
0x6BBBD0D0U, 0x2AC5EFEFU, 0xE54FAAAAU, 0x16EDFBFBU, 0xC5864343U, 0xD79A4D4DU, 0x55663333U, 0x94118585U,
0xCF8A4545U, 0x10E9F9F9U, 0x06040202U, 0x81FE7F7FU, 0xF0A05050U, 0x44783C3CU, 0xBA259F9FU, 0xE34BA8A8U,
0xF3A25151U, 0xFE5DA3A3U, 0xC0804040U, 0x8A058F8FU, 0xAD3F9292U, 0xBC219D9DU, 0x48703838U, 0x04F1F5F5U,
0xDF63BCBCU, 0xC177B6B6U, 0x75AFDADAU, 0x63422121U, 0x30201010U, 0x1AE5FFFFU, 0x0EFDF3F3U, 0x6DBFD2D2U,
0x4C81CDCDU, 0x14180C0CU, 0x35261313U, 0x2FC3ECECU, 0xE1BE5F5FU, 0xA2359797U, 0xCC884444U, 0x392E1717U,
0x5793C4C4U, 0xF255A7A7U, 0x82FC7E7EU, 0x477A3D3DU, 0xACC86464U, 0xE7BA5D5DU, 0x2B321919U, 0x95E67373U,
0xA0C06060U, 0x98198181U, 0xD19E4F4FU, 0x7FA3DCDCU, 0x66442222U, 0x7E542A2AU, 0xAB3B9090U, 0x830B8888U,
0xCA8C4646U, 0x29C7EEEEU, 0xD36BB8B8U, 0x3C281414U, 0x79A7DEDEU, 0xE2BC5E5EU, 0x1D160B0BU, 0x76ADDBDBU,
0x3BDBE0E0U, 0x56643232U, 0x4E743A3AU, 0x1E140A0AU, 0xDB924949U, 0x0A0C0606U, 0x6C482424U, 0xE4B85C5CU,
0x5D9FC2C2U, 0x6EBDD3D3U, 0xEF43ACACU, 0xA6C46262U, 0xA8399191U, 0xA4319595U, 0x37D3E4E4U, 0x8BF27979U,
0x32D5E7E7U, 0x438BC8C8U, 0x596E3737U, 0xB7DA6D6DU, 0x8C018D8DU, 0x64B1D5D5U, 0xD29C4E4EU, 0xE049A9A9U,
0xB4D86C6CU, 0xFAAC5656U, 0x07F3F4F4U, 0x25CFEAEAU, 0xAFCA6565U, 0x8EF47A7AU, 0xE947AEAEU, 0x18100808U,
0xD56FBABAU, 0x88F07878U, 0x6F4A2525U, 0x725C2E2EU, 0x24381C1CU, 0xF157A6A6U, 0xC773B4B4U, 0x5197C6C6U,
0x23CBE8E8U, 0x7CA1DDDDU, 0x9CE87474U, 0x213E1F1FU, 0xDD964B4BU, 0xDC61BDBDU, 0x860D8B8BU, 0x850F8A8AU,
0x90E07070U, 0x427C3E3EU, 0xC471B5B5U, 0xAACC6666U, 0xD8904848U, 0x05060303U, 0x01F7F6F6U, 0x121C0E0EU,
0xA3C26161U, 0x5F6A3535U, 0xF9AE5757U, 0xD069B9B9U, 0x91178686U, 0x5899C1C1U, 0x273A1D1DU, 0xB9279E9EU,
0x38D9E1E1U, 0x13EBF8F8U, 0xB32B9898U, 0x33221111U, 0xBBD26969U, 0x70A9D9D9U, 0x89078E8EU, 0xA7339494U,
0xB62D9B9BU, 0x223C1E1EU, 0x92158787U, 0x20C9E9E9U, 0x4987CECEU, 0xFFAA5555U, 0x78502828U, 0x7AA5DFDFU,
0x8F038C8CU, 0xF859A1A1U, 0x80098989U, 0x171A0D0DU, 0xDA65BFBFU, 0x31D7E6E6U, 0xC6844242U, 0xB8D06868U,
0xC3824141U, 0xB0299999U, 0x775A2D2DU, 0x111E0F0FU, 0xCB7BB0B0U, 0xFCA85454U, 0xD66DBBBBU, 0x3A2C1616U
};
const uint32_t AES::Te2[256] = {
0x63A5C663U, 0x7C84F87CU, 0x7799EE77U, 0x7B8DF67BU, 0xF20DFFF2U, 0x6BBDD66BU, 0x6FB1DE6FU, 0xC55491C5U,
0x30506030U, 0x01030201U, 0x67A9CE67U, 0x2B7D562BU, 0xFE19E7FEU, 0xD762B5D7U, 0xABE64DABU, 0x769AEC76U,
0xCA458FCAU, 0x829D1F82U, 0xC94089C9U, 0x7D87FA7DU, 0xFA15EFFAU, 0x59EBB259U, 0x47C98E47U, 0xF00BFBF0U,
0xADEC41ADU, 0xD467B3D4U, 0xA2FD5FA2U, 0xAFEA45AFU, 0x9CBF239CU, 0xA4F753A4U, 0x7296E472U, 0xC05B9BC0U,
0xB7C275B7U, 0xFD1CE1FDU, 0x93AE3D93U, 0x266A4C26U, 0x365A6C36U, 0x3F417E3FU, 0xF702F5F7U, 0xCC4F83CCU,
0x345C6834U, 0xA5F451A5U, 0xE534D1E5U, 0xF108F9F1U, 0x7193E271U, 0xD873ABD8U, 0x31536231U, 0x153F2A15U,
0x040C0804U, 0xC75295C7U, 0x23654623U, 0xC35E9DC3U, 0x18283018U, 0x96A13796U, 0x050F0A05U, 0x9AB52F9AU,
0x07090E07U, 0x12362412U, 0x809B1B80U, 0xE23DDFE2U, 0xEB26CDEBU, 0x27694E27U, 0xB2CD7FB2U, 0x759FEA75U,
0x091B1209U, 0x839E1D83U, 0x2C74582CU, 0x1A2E341AU, 0x1B2D361BU, 0x6EB2DC6EU, 0x5AEEB45AU, 0xA0FB5BA0U,
0x52F6A452U, 0x3B4D763BU, 0xD661B7D6U, 0xB3CE7DB3U, 0x297B5229U, 0xE33EDDE3U, 0x2F715E2FU, 0x84971384U,
0x53F5A653U, 0xD168B9D1U, 0x00000000U, 0xED2CC1EDU, 0x20604020U, 0xFC1FE3FCU, 0xB1C879B1U, 0x5BEDB65BU,
0x6ABED46AU, 0xCB468DCBU, 0xBED967BEU, 0x394B7239U, 0x4ADE944AU, 0x4CD4984CU, 0x58E8B058U, 0xCF4A85CFU,
0xD06BBBD0U, 0xEF2AC5EFU, 0xAAE54FAAU, 0xFB16EDFBU, 0x43C58643U, 0x4DD79A4DU, 0x33556633U, 0x85941185U,
0x45CF8A45U, 0xF910E9F9U, 0x02060402U, 0x7F81FE7FU, 0x50F0A050U, 0x3C44783CU, 0x9FBA259FU, 0xA8E34BA8U,
0x51F3A251U, 0xA3FE5DA3U, 0x40C08040U, 0x8F8A058FU, 0x92AD3F92U, 0x9DBC219DU, 0x38487038U, 0xF504F1F5U,
0xBCDF63BCU, 0xB6C177B6U, 0xDA75AFDAU, 0x21634221U, 0x10302010U, 0xFF1AE5FFU, 0xF30EFDF3U, 0xD26DBFD2U,
0xCD4C81CDU, 0x0C14180CU, 0x13352613U, 0xEC2FC3ECU, 0x5FE1BE5FU, 0x97A23597U, 0x44CC8844U, 0x17392E17U,
0xC45793C4U, 0xA7F255A7U, 0x7E82FC7EU, 0x3D477A3DU, 0x64ACC864U, 0x5DE7BA5DU, 0x192B3219U, 0x7395E673U,
0x60A0C060U, 0x81981981U, 0x4FD19E4FU, 0xDC7FA3DCU, 0x22664422U, 0x2A7E542AU, 0x90AB3B90U, 0x88830B88U,
0x46CA8C46U, 0xEE29C7EEU, 0xB8D36BB8U, 0x143C2814U, 0xDE79A7DEU, 0x5EE2BC5EU, 0x0B1D160BU, 0xDB76ADDBU,
0xE03BDBE0U, 0x32566432U, 0x3A4E743AU, 0x0A1E140AU, 0x49DB9249U, 0x060A0C06U, 0x246C4824U, 0x5CE4B85CU,
0xC25D9FC2U, 0xD36EBDD3U, 0xACEF43ACU, 0x62A6C462U, 0x91A83991U, 0x95A43195U, 0xE437D3E4U, 0x798BF279U,
0xE732D5E7U, 0xC8438BC8U, 0x37596E37U, 0x6DB7DA6DU, 0x8D8C018DU, 0xD564B1D5U, 0x4ED29C4EU, 0xA9E049A9U,
0x6CB4D86CU, 0x56FAAC56U, 0xF407F3F4U, 0xEA25CFEAU, 0x65AFCA65U, 0x7A8EF47AU, 0xAEE947AEU, 0x08181008U,
0xBAD56FBAU, 0x7888F078U, 0x256F4A25U, 0x2E725C2EU, 0x1C24381CU, 0xA6F157A6U, 0xB4C773B4U, 0xC65197C6U,
0xE823CBE8U, 0xDD7CA1DDU, 0x749CE874U, 0x1F213E1FU, 0x4BDD964BU, 0xBDDC61BDU, 0x8B860D8BU, 0x8A850F8AU,
0x7090E070U, 0x3E427C3EU, 0xB5C471B5U, 0x66AACC66U, 0x48D89048U, 0x03050603U, 0xF601F7F6U, 0x0E121C0EU,
0x61A3C261U, 0x355F6A35U, 0x57F9AE57U, 0xB9D069B9U, 0x86911786U, 0xC15899C1U, 0x1D273A1DU, 0x9EB9279EU,
0xE138D9E1U, 0xF813EBF8U, 0x98B32B98U, 0x11332211U, 0x69BBD269U, 0xD970A9D9U, 0x8E89078EU, 0x94A73394U,
0x9BB62D9BU, 0x1E223C1EU, 0x87921587U, 0xE920C9E9U, 0xCE4987CEU, 0x55FFAA55U, 0x28785028U, 0xDF7AA5DFU,
0x8C8F038CU, 0xA1F859A1U, 0x89800989U, 0x0D171A0DU, 0xBFDA65BFU, 0xE631D7E6U, 0x42C68442U, 0x68B8D068U,
0x41C38241U, 0x99B02999U, 0x2D775A2DU, 0x0F111E0FU, 0xB0CB7BB0U, 0x54FCA854U, 0xBBD66DBBU, 0x163A2C16U
};
const uint32_t AES::Te3[256] = {
0x6363A5C6U, 0x7C7C84F8U, 0x777799EEU, 0x7B7B8DF6U, 0xF2F20DFFU, 0x6B6BBDD6U, 0x6F6FB1DEU, 0xC5C55491U,
0x30305060U, 0x01010302U, 0x6767A9CEU, 0x2B2B7D56U, 0xFEFE19E7U, 0xD7D762B5U, 0xABABE64DU, 0x76769AECU,
0xCACA458FU, 0x82829D1FU, 0xC9C94089U, 0x7D7D87FAU, 0xFAFA15EFU, 0x5959EBB2U, 0x4747C98EU, 0xF0F00BFBU,
0xADADEC41U, 0xD4D467B3U, 0xA2A2FD5FU, 0xAFAFEA45U, 0x9C9CBF23U, 0xA4A4F753U, 0x727296E4U, 0xC0C05B9BU,
0xB7B7C275U, 0xFDFD1CE1U, 0x9393AE3DU, 0x26266A4CU, 0x36365A6CU, 0x3F3F417EU, 0xF7F702F5U, 0xCCCC4F83U,
0x34345C68U, 0xA5A5F451U, 0xE5E534D1U, 0xF1F108F9U, 0x717193E2U, 0xD8D873ABU, 0x31315362U, 0x15153F2AU,
0x04040C08U, 0xC7C75295U, 0x23236546U, 0xC3C35E9DU, 0x18182830U, 0x9696A137U, 0x05050F0AU, 0x9A9AB52FU,
0x0707090EU, 0x12123624U, 0x80809B1BU, 0xE2E23DDFU, 0xEBEB26CDU, 0x2727694EU, 0xB2B2CD7FU, 0x75759FEAU,
0x09091B12U, 0x83839E1DU, 0x2C2C7458U, 0x1A1A2E34U, 0x1B1B2D36U, 0x6E6EB2DCU, 0x5A5AEEB4U, 0xA0A0FB5BU,
0x5252F6A4U, 0x3B3B4D76U, 0xD6D661B7U, 0xB3B3CE7DU, 0x29297B52U, 0xE3E33EDDU, 0x2F2F715EU, 0x84849713U,
0x5353F5A6U, 0xD1D168B9U, 0x00000000U, 0xEDED2CC1U, 0x20206040U, 0xFCFC1FE3U, 0xB1B1C879U, 0x5B5BEDB6U,
0x6A6ABED4U, 0xCBCB468DU, 0xBEBED967U, 0x39394B72U, 0x4A4ADE94U, 0x4C4CD498U, 0x5858E8B0U, 0xCFCF4A85U,
0xD0D06BBBU, 0xEFEF2AC5U, 0xAAAAE54FU, 0xFBFB16EDU, 0x4343C586U, 0x4D4DD79AU, 0x33335566U, 0x85859411U,
0x4545CF8AU, 0xF9F910E9U, 0x02020604U, 0x7F7F81FEU, 0x5050F0A0U, 0x3C3C4478U, 0x9F9FBA25U, 0xA8A8E34BU,
0x5151F3A2U, 0xA3A3FE5DU, 0x4040C080U, 0x8F8F8A05U, 0x9292AD3FU, 0x9D9DBC21U, 0x38384870U, 0xF5F504F1U,
0xBCBCDF63U, 0xB6B6C177U, 0xDADA75AFU, 0x21216342U, 0x10103020U, 0xFFFF1AE5U, 0xF3F30EFDU, 0xD2D26DBFU,
0xCDCD4C81U, 0x0C0C1418U, 0x13133526U, 0xECEC2FC3U, 0x5F5FE1BEU, 0x9797A235U, 0x4444CC88U, 0x1717392EU,
0xC4C45793U, 0xA7A7F255U, 0x7E7E82FCU, 0x3D3D477AU, 0x6464ACC8U, 0x5D5DE7BAU, 0x19192B32U, 0x737395E6U,
0x6060A0C0U, 0x81819819U, 0x4F4FD19EU, 0xDCDC7FA3U, 0x22226644U, 0x2A2A7E54U, 0x9090AB3BU, 0x8888830BU,
0x4646CA8CU, 0xEEEE29C7U, 0xB8B8D36BU, 0x14143C28U, 0xDEDE79A7U, 0x5E5EE2BCU, 0x0B0B1D16U, 0xDBDB76ADU,
0xE0E03BDBU, 0x32325664U, 0x3A3A4E74U, 0x0A0A1E14U, 0x4949DB92U, 0x06060A0CU, 0x24246C48U, 0x5C5CE4B8U,
0xC2C25D9FU, 0xD3D36EBDU, 0xACACEF43U, 0x6262A6C4U, 0x9191A839U, 0x9595A431U, 0xE4E437D3U, 0x79798BF2U,
0xE7E732D5U, 0xC8C8438BU, 0x3737596EU, 0x6D6DB7DAU, 0x8D8D8C01U, 0xD5D564B1U, 0x4E4ED29CU, 0xA9A9E049U,
0x6C6CB4D8U, 0x5656FAACU, 0xF4F407F3U, 0xEAEA25CFU, 0x6565AFCAU, 0x7A7A8EF4U, 0xAEAEE947U, 0x08081810U,
0xBABAD56FU, 0x787888F0U, 0x25256F4AU, 0x2E2E725CU, 0x1C1C2438U, 0xA6A6F157U, 0xB4B4C773U, 0xC6C65197U,
0xE8E823CBU, 0xDDDD7CA1U, 0x74749CE8U, 0x1F1F213EU, 0x4B4BDD96U, 0xBDBDDC61U, 0x8B8B860DU, 0x8A8A850FU,
0x707090E0U, 0x3E3E427CU, 0xB5B5C471U, 0x6666AACCU, 0x4848D890U, 0x03030506U, 0xF6F601F7U, 0x0E0E121CU,
0x6161A3C2U, 0x35355F6AU, 0x5757F9AEU, 0xB9B9D069U, 0x86869117U, 0xC1C15899U, 0x1D1D273AU, 0x9E9EB927U,
0xE1E138D9U, 0xF8F813EBU, 0x9898B32BU, 0x11113322U, 0x6969BBD2U, 0xD9D970A9U, 0x8E8E8907U, 0x9494A733U,
0x9B9BB62DU, 0x1E1E223CU, 0x87879215U, 0xE9E920C9U, 0xCECE4987U, 0x5555FFAAU, 0x28287850U, 0xDFDF7AA5U,
0x8C8C8F03U, 0xA1A1F859U, 0x89898009U, 0x0D0D171AU, 0xBFBFDA65U, 0xE6E631D7U, 0x4242C684U, 0x6868B8D0U,
0x4141C382U, 0x9999B029U, 0x2D2D775AU, 0x0F0F111EU, 0xB0B0CB7BU, 0x5454FCA8U, 0xBBBBD66DU, 0x16163A2CU
};
const uint32_t AES::Te4[256] = {
0x63636363U, 0x7C7C7C7CU, 0x77777777U, 0x7B7B7B7BU, 0xF2F2F2F2U, 0x6B6B6B6BU, 0x6F6F6F6FU, 0xC5C5C5C5U,
0x30303030U, 0x01010101U, 0x67676767U, 0x2B2B2B2BU, 0xFEFEFEFEU, 0xD7D7D7D7U, 0xABABABABU, 0x76767676U,
0xCACACACAU, 0x82828282U, 0xC9C9C9C9U, 0x7D7D7D7DU, 0xFAFAFAFAU, 0x59595959U, 0x47474747U, 0xF0F0F0F0U,
0xADADADADU, 0xD4D4D4D4U, 0xA2A2A2A2U, 0xAFAFAFAFU, 0x9C9C9C9CU, 0xA4A4A4A4U, 0x72727272U, 0xC0C0C0C0U,
0xB7B7B7B7U, 0xFDFDFDFDU, 0x93939393U, 0x26262626U, 0x36363636U, 0x3F3F3F3FU, 0xF7F7F7F7U, 0xCCCCCCCCU,
0x34343434U, 0xA5A5A5A5U, 0xE5E5E5E5U, 0xF1F1F1F1U, 0x71717171U, 0xD8D8D8D8U, 0x31313131U, 0x15151515U,
0x04040404U, 0xC7C7C7C7U, 0x23232323U, 0xC3C3C3C3U, 0x18181818U, 0x96969696U, 0x05050505U, 0x9A9A9A9AU,
0x07070707U, 0x12121212U, 0x80808080U, 0xE2E2E2E2U, 0xEBEBEBEBU, 0x27272727U, 0xB2B2B2B2U, 0x75757575U,
0x09090909U, 0x83838383U, 0x2C2C2C2CU, 0x1A1A1A1AU, 0x1B1B1B1BU, 0x6E6E6E6EU, 0x5A5A5A5AU, 0xA0A0A0A0U,
0x52525252U, 0x3B3B3B3BU, 0xD6D6D6D6U, 0xB3B3B3B3U, 0x29292929U, 0xE3E3E3E3U, 0x2F2F2F2FU, 0x84848484U,
0x53535353U, 0xD1D1D1D1U, 0x00000000U, 0xEDEDEDEDU, 0x20202020U, 0xFCFCFCFCU, 0xB1B1B1B1U, 0x5B5B5B5BU,
0x6A6A6A6AU, 0xCBCBCBCBU, 0xBEBEBEBEU, 0x39393939U, 0x4A4A4A4AU, 0x4C4C4C4CU, 0x58585858U, 0xCFCFCFCFU,
0xD0D0D0D0U, 0xEFEFEFEFU, 0xAAAAAAAAU, 0xFBFBFBFBU, 0x43434343U, 0x4D4D4D4DU, 0x33333333U, 0x85858585U,
0x45454545U, 0xF9F9F9F9U, 0x02020202U, 0x7F7F7F7FU, 0x50505050U, 0x3C3C3C3CU, 0x9F9F9F9FU, 0xA8A8A8A8U,
0x51515151U, 0xA3A3A3A3U, 0x40404040U, 0x8F8F8F8FU, 0x92929292U, 0x9D9D9D9DU, 0x38383838U, 0xF5F5F5F5U,
0xBCBCBCBCU, 0xB6B6B6B6U, 0xDADADADAU, 0x21212121U, 0x10101010U, 0xFFFFFFFFU, 0xF3F3F3F3U, 0xD2D2D2D2U,
0xCDCDCDCDU, 0x0C0C0C0CU, 0x13131313U, 0xECECECECU, 0x5F5F5F5FU, 0x97979797U, 0x44444444U, 0x17171717U,
0xC4C4C4C4U, 0xA7A7A7A7U, 0x7E7E7E7EU, 0x3D3D3D3DU, 0x64646464U, 0x5D5D5D5DU, 0x19191919U, 0x73737373U,
0x60606060U, 0x81818181U, 0x4F4F4F4FU, 0xDCDCDCDCU, 0x22222222U, 0x2A2A2A2AU, 0x90909090U, 0x88888888U,
0x46464646U, 0xEEEEEEEEU, 0xB8B8B8B8U, 0x14141414U, 0xDEDEDEDEU, 0x5E5E5E5EU, 0x0B0B0B0BU, 0xDBDBDBDBU,
0xE0E0E0E0U, 0x32323232U, 0x3A3A3A3AU, 0x0A0A0A0AU, 0x49494949U, 0x06060606U, 0x24242424U, 0x5C5C5C5CU,
0xC2C2C2C2U, 0xD3D3D3D3U, 0xACACACACU, 0x62626262U, 0x91919191U, 0x95959595U, 0xE4E4E4E4U, 0x79797979U,
0xE7E7E7E7U, 0xC8C8C8C8U, 0x37373737U, 0x6D6D6D6DU, 0x8D8D8D8DU, 0xD5D5D5D5U, 0x4E4E4E4EU, 0xA9A9A9A9U,
0x6C6C6C6CU, 0x56565656U, 0xF4F4F4F4U, 0xEAEAEAEAU, 0x65656565U, 0x7A7A7A7AU, 0xAEAEAEAEU, 0x08080808U,
0xBABABABAU, 0x78787878U, 0x25252525U, 0x2E2E2E2EU, 0x1C1C1C1CU, 0xA6A6A6A6U, 0xB4B4B4B4U, 0xC6C6C6C6U,
0xE8E8E8E8U, 0xDDDDDDDDU, 0x74747474U, 0x1F1F1F1FU, 0x4B4B4B4BU, 0xBDBDBDBDU, 0x8B8B8B8BU, 0x8A8A8A8AU,
0x70707070U, 0x3E3E3E3EU, 0xB5B5B5B5U, 0x66666666U, 0x48484848U, 0x03030303U, 0xF6F6F6F6U, 0x0E0E0E0EU,
0x61616161U, 0x35353535U, 0x57575757U, 0xB9B9B9B9U, 0x86868686U, 0xC1C1C1C1U, 0x1D1D1D1DU, 0x9E9E9E9EU,
0xE1E1E1E1U, 0xF8F8F8F8U, 0x98989898U, 0x11111111U, 0x69696969U, 0xD9D9D9D9U, 0x8E8E8E8EU, 0x94949494U,
0x9B9B9B9BU, 0x1E1E1E1EU, 0x87878787U, 0xE9E9E9E9U, 0xCECECECEU, 0x55555555U, 0x28282828U, 0xDFDFDFDFU,
0x8C8C8C8CU, 0xA1A1A1A1U, 0x89898989U, 0x0D0D0D0DU, 0xBFBFBFBFU, 0xE6E6E6E6U, 0x42424242U, 0x68686868U,
0x41414141U, 0x99999999U, 0x2D2D2D2DU, 0x0F0F0F0FU, 0xB0B0B0B0U, 0x54545454U, 0xBBBBBBBBU, 0x16161616U
};
const uint32_t AES::Td0[256] = {
0x51F4A750U, 0x7E416553U, 0x1A17A4C3U, 0x3A275E96U, 0x3BAB6BCBU, 0x1F9D45F1U, 0xACFA58ABU, 0x4BE30393U,
0x2030FA55U, 0xAD766DF6U, 0x88CC7691U, 0xF5024C25U, 0x4FE5D7FCU, 0xC52ACBD7U, 0x26354480U, 0xB562A38FU,
0xDEB15A49U, 0x25BA1B67U, 0x45EA0E98U, 0x5DFEC0E1U, 0xC32F7502U, 0x814CF012U, 0x8D4697A3U, 0x6BD3F9C6U,
0x038F5FE7U, 0x15929C95U, 0xBF6D7AEBU, 0x955259DAU, 0xD4BE832DU, 0x587421D3U, 0x49E06929U, 0x8EC9C844U,
0x75C2896AU, 0xF48E7978U, 0x99583E6BU, 0x27B971DDU, 0xBEE14FB6U, 0xF088AD17U, 0xC920AC66U, 0x7DCE3AB4U,
0x63DF4A18U, 0xE51A3182U, 0x97513360U, 0x62537F45U, 0xB16477E0U, 0xBB6BAE84U, 0xFE81A01CU, 0xF9082B94U,
0x70486858U, 0x8F45FD19U, 0x94DE6C87U, 0x527BF8B7U, 0xAB73D323U, 0x724B02E2U, 0xE31F8F57U, 0x6655AB2AU,
0xB2EB2807U, 0x2FB5C203U, 0x86C57B9AU, 0xD33708A5U, 0x302887F2U, 0x23BFA5B2U, 0x02036ABAU, 0xED16825CU,
0x8ACF1C2BU, 0xA779B492U, 0xF307F2F0U, 0x4E69E2A1U, 0x65DAF4CDU, 0x0605BED5U, 0xD134621FU, 0xC4A6FE8AU,
0x342E539DU, 0xA2F355A0U, 0x058AE132U, 0xA4F6EB75U, 0x0B83EC39U, 0x4060EFAAU, 0x5E719F06U, 0xBD6E1051U,
0x3E218AF9U, 0x96DD063DU, 0xDD3E05AEU, 0x4DE6BD46U, 0x91548DB5U, 0x71C45D05U, 0x0406D46FU, 0x605015FFU,
0x1998FB24U, 0xD6BDE997U, 0x894043CCU, 0x67D99E77U, 0xB0E842BDU, 0x07898B88U, 0xE7195B38U, 0x79C8EEDBU,
0xA17C0A47U, 0x7C420FE9U, 0xF8841EC9U, 0x00000000U, 0x09808683U, 0x322BED48U, 0x1E1170ACU, 0x6C5A724EU,
0xFD0EFFFBU, 0x0F853856U, 0x3DAED51EU, 0x362D3927U, 0x0A0FD964U, 0x685CA621U, 0x9B5B54D1U, 0x24362E3AU,
0x0C0A67B1U, 0x9357E70FU, 0xB4EE96D2U, 0x1B9B919EU, 0x80C0C54FU, 0x61DC20A2U, 0x5A774B69U, 0x1C121A16U,
0xE293BA0AU, 0xC0A02AE5U, 0x3C22E043U, 0x121B171DU, 0x0E090D0BU, 0xF28BC7ADU, 0x2DB6A8B9U, 0x141EA9C8U,
0x57F11985U, 0xAF75074CU, 0xEE99DDBBU, 0xA37F60FDU, 0xF701269FU, 0x5C72F5BCU, 0x44663BC5U, 0x5BFB7E34U,
0x8B432976U, 0xCB23C6DCU, 0xB6EDFC68U, 0xB8E4F163U, 0xD731DCCAU, 0x42638510U, 0x13972240U, 0x84C61120U,
0x854A247DU, 0xD2BB3DF8U, 0xAEF93211U, 0xC729A16DU, 0x1D9E2F4BU, 0xDCB230F3U, 0x0D8652ECU, 0x77C1E3D0U,
0x2BB3166CU, 0xA970B999U, 0x119448FAU, 0x47E96422U, 0xA8FC8CC4U, 0xA0F03F1AU, 0x567D2CD8U, 0x223390EFU,
0x87494EC7U, 0xD938D1C1U, 0x8CCAA2FEU, 0x98D40B36U, 0xA6F581CFU, 0xA57ADE28U, 0xDAB78E26U, 0x3FADBFA4U,
0x2C3A9DE4U, 0x5078920DU, 0x6A5FCC9BU, 0x547E4662U, 0xF68D13C2U, 0x90D8B8E8U, 0x2E39F75EU, 0x82C3AFF5U,
0x9F5D80BEU, 0x69D0937CU, 0x6FD52DA9U, 0xCF2512B3U, 0xC8AC993BU, 0x10187DA7U, 0xE89C636EU, 0xDB3BBB7BU,
0xCD267809U, 0x6E5918F4U, 0xEC9AB701U, 0x834F9AA8U, 0xE6956E65U, 0xAAFFE67EU, 0x21BCCF08U, 0xEF15E8E6U,
0xBAE79BD9U, 0x4A6F36CEU, 0xEA9F09D4U, 0x29B07CD6U, 0x31A4B2AFU, 0x2A3F2331U, 0xC6A59430U, 0x35A266C0U,
0x744EBC37U, 0xFC82CAA6U, 0xE090D0B0U, 0x33A7D815U, 0xF104984AU, 0x41ECDAF7U, 0x7FCD500EU, 0x1791F62FU,
0x764DD68DU, 0x43EFB04DU, 0xCCAA4D54U, 0xE49604DFU, 0x9ED1B5E3U, 0x4C6A881BU, 0xC12C1FB8U, 0x4665517FU,
0x9D5EEA04U, 0x018C355DU, 0xFA877473U, 0xFB0B412EU, 0xB3671D5AU, 0x92DBD252U, 0xE9105633U, 0x6DD64713U,
0x9AD7618CU, 0x37A10C7AU, 0x59F8148EU, 0xEB133C89U, 0xCEA927EEU, 0xB761C935U, 0xE11CE5EDU, 0x7A47B13CU,
0x9CD2DF59U, 0x55F2733FU, 0x1814CE79U, 0x73C737BFU, 0x53F7CDEAU, 0x5FFDAA5BU, 0xDF3D6F14U, 0x7844DB86U,
0xCAAFF381U, 0xB968C43EU, 0x3824342CU, 0xC2A3405FU, 0x161DC372U, 0xBCE2250CU, 0x283C498BU, 0xFF0D9541U,
0x39A80171U, 0x080CB3DEU, 0xD8B4E49CU, 0x6456C190U, 0x7BCB8461U, 0xD532B670U, 0x486C5C74U, 0xD0B85742U
};
const uint32_t AES::Td1[256] = {
0x5051F4A7U, 0x537E4165U, 0xC31A17A4U, 0x963A275EU, 0xCB3BAB6BU, 0xF11F9D45U, 0xABACFA58U, 0x934BE303U,
0x552030FAU, 0xF6AD766DU, 0x9188CC76U, 0x25F5024CU, 0xFC4FE5D7U, 0xD7C52ACBU, 0x80263544U, 0x8FB562A3U,
0x49DEB15AU, 0x6725BA1BU, 0x9845EA0EU, 0xE15DFEC0U, 0x02C32F75U, 0x12814CF0U, 0xA38D4697U, 0xC66BD3F9U,
0xE7038F5FU, 0x9515929CU, 0xEBBF6D7AU, 0xDA955259U, 0x2DD4BE83U, 0xD3587421U, 0x2949E069U, 0x448EC9C8U,
0x6A75C289U, 0x78F48E79U, 0x6B99583EU, 0xDD27B971U, 0xB6BEE14FU, 0x17F088ADU, 0x66C920ACU, 0xB47DCE3AU,
0x1863DF4AU, 0x82E51A31U, 0x60975133U, 0x4562537FU, 0xE0B16477U, 0x84BB6BAEU, 0x1CFE81A0U, 0x94F9082BU,
0x58704868U, 0x198F45FDU, 0x8794DE6CU, 0xB7527BF8U, 0x23AB73D3U, 0xE2724B02U, 0x57E31F8FU, 0x2A6655ABU,
0x07B2EB28U, 0x032FB5C2U, 0x9A86C57BU, 0xA5D33708U, 0xF2302887U, 0xB223BFA5U, 0xBA02036AU, 0x5CED1682U,
0x2B8ACF1CU, 0x92A779B4U, 0xF0F307F2U, 0xA14E69E2U, 0xCD65DAF4U, 0xD50605BEU, 0x1FD13462U, 0x8AC4A6FEU,
0x9D342E53U, 0xA0A2F355U, 0x32058AE1U, 0x75A4F6EBU, 0x390B83ECU, 0xAA4060EFU, 0x065E719FU, 0x51BD6E10U,
0xF93E218AU, 0x3D96DD06U, 0xAEDD3E05U, 0x464DE6BDU, 0xB591548DU, 0x0571C45DU, 0x6F0406D4U, 0xFF605015U,
0x241998FBU, 0x97D6BDE9U, 0xCC894043U, 0x7767D99EU, 0xBDB0E842U, 0x8807898BU, 0x38E7195BU, 0xDB79C8EEU,
0x47A17C0AU, 0xE97C420FU, 0xC9F8841EU, 0x00000000U, 0x83098086U, 0x48322BEDU, 0xAC1E1170U, 0x4E6C5A72U,
0xFBFD0EFFU, 0x560F8538U, 0x1E3DAED5U, 0x27362D39U, 0x640A0FD9U, 0x21685CA6U, 0xD19B5B54U, 0x3A24362EU,
0xB10C0A67U, 0x0F9357E7U, 0xD2B4EE96U, 0x9E1B9B91U, 0x4F80C0C5U, 0xA261DC20U, 0x695A774BU, 0x161C121AU,
0x0AE293BAU, 0xE5C0A02AU, 0x433C22E0U, 0x1D121B17U, 0x0B0E090DU, 0xADF28BC7U, 0xB92DB6A8U, 0xC8141EA9U,
0x8557F119U, 0x4CAF7507U, 0xBBEE99DDU, 0xFDA37F60U, 0x9FF70126U, 0xBC5C72F5U, 0xC544663BU, 0x345BFB7EU,
0x768B4329U, 0xDCCB23C6U, 0x68B6EDFCU, 0x63B8E4F1U, 0xCAD731DCU, 0x10426385U, 0x40139722U, 0x2084C611U,
0x7D854A24U, 0xF8D2BB3DU, 0x11AEF932U, 0x6DC729A1U, 0x4B1D9E2FU, 0xF3DCB230U, 0xEC0D8652U, 0xD077C1E3U,
0x6C2BB316U, 0x99A970B9U, 0xFA119448U, 0x2247E964U, 0xC4A8FC8CU, 0x1AA0F03FU, 0xD8567D2CU, 0xEF223390U,
0xC787494EU, 0xC1D938D1U, 0xFE8CCAA2U, 0x3698D40BU, 0xCFA6F581U, 0x28A57ADEU, 0x26DAB78EU, 0xA43FADBFU,
0xE42C3A9DU, 0x0D507892U, 0x9B6A5FCCU, 0x62547E46U, 0xC2F68D13U, 0xE890D8B8U, 0x5E2E39F7U, 0xF582C3AFU,
0xBE9F5D80U, 0x7C69D093U, 0xA96FD52DU, 0xB3CF2512U, 0x3BC8AC99U, 0xA710187DU, 0x6EE89C63U, 0x7BDB3BBBU,
0x09CD2678U, 0xF46E5918U, 0x01EC9AB7U, 0xA8834F9AU, 0x65E6956EU, 0x7EAAFFE6U, 0x0821BCCFU, 0xE6EF15E8U,
0xD9BAE79BU, 0xCE4A6F36U, 0xD4EA9F09U, 0xD629B07CU, 0xAF31A4B2U, 0x312A3F23U, 0x30C6A594U, 0xC035A266U,
0x37744EBCU, 0xA6FC82CAU, 0xB0E090D0U, 0x1533A7D8U, 0x4AF10498U, 0xF741ECDAU, 0x0E7FCD50U, 0x2F1791F6U,
0x8D764DD6U, 0x4D43EFB0U, 0x54CCAA4DU, 0xDFE49604U, 0xE39ED1B5U, 0x1B4C6A88U, 0xB8C12C1FU, 0x7F466551U,
0x049D5EEAU, 0x5D018C35U, 0x73FA8774U, 0x2EFB0B41U, 0x5AB3671DU, 0x5292DBD2U, 0x33E91056U, 0x136DD647U,
0x8C9AD761U, 0x7A37A10CU, 0x8E59F814U, 0x89EB133CU, 0xEECEA927U, 0x35B761C9U, 0xEDE11CE5U, 0x3C7A47B1U,
0x599CD2DFU, 0x3F55F273U, 0x791814CEU, 0xBF73C737U, 0xEA53F7CDU, 0x5B5FFDAAU, 0x14DF3D6FU, 0x867844DBU,
0x81CAAFF3U, 0x3EB968C4U, 0x2C382434U, 0x5FC2A340U, 0x72161DC3U, 0x0CBCE225U, 0x8B283C49U, 0x41FF0D95U,
0x7139A801U, 0xDE080CB3U, 0x9CD8B4E4U, 0x906456C1U, 0x617BCB84U, 0x70D532B6U, 0x74486C5CU, 0x42D0B857U
};
const uint32_t AES::Td2[256] = {
0xA75051F4U, 0x65537E41U, 0xA4C31A17U, 0x5E963A27U, 0x6BCB3BABU, 0x45F11F9DU, 0x58ABACFAU, 0x03934BE3U,
0xFA552030U, 0x6DF6AD76U, 0x769188CCU, 0x4C25F502U, 0xD7FC4FE5U, 0xCBD7C52AU, 0x44802635U, 0xA38FB562U,
0x5A49DEB1U, 0x1B6725BAU, 0x0E9845EAU, 0xC0E15DFEU, 0x7502C32FU, 0xF012814CU, 0x97A38D46U, 0xF9C66BD3U,
0x5FE7038FU, 0x9C951592U, 0x7AEBBF6DU, 0x59DA9552U, 0x832DD4BEU, 0x21D35874U, 0x692949E0U, 0xC8448EC9U,
0x896A75C2U, 0x7978F48EU, 0x3E6B9958U, 0x71DD27B9U, 0x4FB6BEE1U, 0xAD17F088U, 0xAC66C920U, 0x3AB47DCEU,
0x4A1863DFU, 0x3182E51AU, 0x33609751U, 0x7F456253U, 0x77E0B164U, 0xAE84BB6BU, 0xA01CFE81U, 0x2B94F908U,
0x68587048U, 0xFD198F45U, 0x6C8794DEU, 0xF8B7527BU, 0xD323AB73U, 0x02E2724BU, 0x8F57E31FU, 0xAB2A6655U,
0x2807B2EBU, 0xC2032FB5U, 0x7B9A86C5U, 0x08A5D337U, 0x87F23028U, 0xA5B223BFU, 0x6ABA0203U, 0x825CED16U,
0x1C2B8ACFU, 0xB492A779U, 0xF2F0F307U, 0xE2A14E69U, 0xF4CD65DAU, 0xBED50605U, 0x621FD134U, 0xFE8AC4A6U,
0x539D342EU, 0x55A0A2F3U, 0xE132058AU, 0xEB75A4F6U, 0xEC390B83U, 0xEFAA4060U, 0x9F065E71U, 0x1051BD6EU,
0x8AF93E21U, 0x063D96DDU, 0x05AEDD3EU, 0xBD464DE6U, 0x8DB59154U, 0x5D0571C4U, 0xD46F0406U, 0x15FF6050U,
0xFB241998U, 0xE997D6BDU, 0x43CC8940U, 0x9E7767D9U, 0x42BDB0E8U, 0x8B880789U, 0x5B38E719U, 0xEEDB79C8U,
0x0A47A17CU, 0x0FE97C42U, 0x1EC9F884U, 0x00000000U, 0x86830980U, 0xED48322BU, 0x70AC1E11U, 0x724E6C5AU,
0xFFFBFD0EU, 0x38560F85U, 0xD51E3DAEU, 0x3927362DU, 0xD9640A0FU, 0xA621685CU, 0x54D19B5BU, 0x2E3A2436U,
0x67B10C0AU, 0xE70F9357U, 0x96D2B4EEU, 0x919E1B9BU, 0xC54F80C0U, 0x20A261DCU, 0x4B695A77U, 0x1A161C12U,
0xBA0AE293U, 0x2AE5C0A0U, 0xE0433C22U, 0x171D121BU, 0x0D0B0E09U, 0xC7ADF28BU, 0xA8B92DB6U, 0xA9C8141EU,
0x198557F1U, 0x074CAF75U, 0xDDBBEE99U, 0x60FDA37FU, 0x269FF701U, 0xF5BC5C72U, 0x3BC54466U, 0x7E345BFBU,
0x29768B43U, 0xC6DCCB23U, 0xFC68B6EDU, 0xF163B8E4U, 0xDCCAD731U, 0x85104263U, 0x22401397U, 0x112084C6U,
0x247D854AU, 0x3DF8D2BBU, 0x3211AEF9U, 0xA16DC729U, 0x2F4B1D9EU, 0x30F3DCB2U, 0x52EC0D86U, 0xE3D077C1U,
0x166C2BB3U, 0xB999A970U, 0x48FA1194U, 0x642247E9U, 0x8CC4A8FCU, 0x3F1AA0F0U, 0x2CD8567DU, 0x90EF2233U,
0x4EC78749U, 0xD1C1D938U, 0xA2FE8CCAU, 0x0B3698D4U, 0x81CFA6F5U, 0xDE28A57AU, 0x8E26DAB7U, 0xBFA43FADU,
0x9DE42C3AU, 0x920D5078U, 0xCC9B6A5FU, 0x4662547EU, 0x13C2F68DU, 0xB8E890D8U, 0xF75E2E39U, 0xAFF582C3U,
0x80BE9F5DU, 0x937C69D0U, 0x2DA96FD5U, 0x12B3CF25U, 0x993BC8ACU, 0x7DA71018U, 0x636EE89CU, 0xBB7BDB3BU,
0x7809CD26U, 0x18F46E59U, 0xB701EC9AU, 0x9AA8834FU, 0x6E65E695U, 0xE67EAAFFU, 0xCF0821BCU, 0xE8E6EF15U,
0x9BD9BAE7U, 0x36CE4A6FU, 0x09D4EA9FU, 0x7CD629B0U, 0xB2AF31A4U, 0x23312A3FU, 0x9430C6A5U, 0x66C035A2U,
0xBC37744EU, 0xCAA6FC82U, 0xD0B0E090U, 0xD81533A7U, 0x984AF104U, 0xDAF741ECU, 0x500E7FCDU, 0xF62F1791U,
0xD68D764DU, 0xB04D43EFU, 0x4D54CCAAU, 0x04DFE496U, 0xB5E39ED1U, 0x881B4C6AU, 0x1FB8C12CU, 0x517F4665U,
0xEA049D5EU, 0x355D018CU, 0x7473FA87U, 0x412EFB0BU, 0x1D5AB367U, 0xD25292DBU, 0x5633E910U, 0x47136DD6U,
0x618C9AD7U, 0x0C7A37A1U, 0x148E59F8U, 0x3C89EB13U, 0x27EECEA9U, 0xC935B761U, 0xE5EDE11CU, 0xB13C7A47U,
0xDF599CD2U, 0x733F55F2U, 0xCE791814U, 0x37BF73C7U, 0xCDEA53F7U, 0xAA5B5FFDU, 0x6F14DF3DU, 0xDB867844U,
0xF381CAAFU, 0xC43EB968U, 0x342C3824U, 0x405FC2A3U, 0xC372161DU, 0x250CBCE2U, 0x498B283CU, 0x9541FF0DU,
0x017139A8U, 0xB3DE080CU, 0xE49CD8B4U, 0xC1906456U, 0x84617BCBU, 0xB670D532U, 0x5C74486CU, 0x5742D0B8U
};
const uint32_t AES::Td3[256] = {
0xF4A75051U, 0x4165537EU, 0x17A4C31AU, 0x275E963AU, 0xAB6BCB3BU, 0x9D45F11FU, 0xFA58ABACU, 0xE303934BU,
0x30FA5520U, 0x766DF6ADU, 0xCC769188U, 0x024C25F5U, 0xE5D7FC4FU, 0x2ACBD7C5U, 0x35448026U, 0x62A38FB5U,
0xB15A49DEU, 0xBA1B6725U, 0xEA0E9845U, 0xFEC0E15DU, 0x2F7502C3U, 0x4CF01281U, 0x4697A38DU, 0xD3F9C66BU,
0x8F5FE703U, 0x929C9515U, 0x6D7AEBBFU, 0x5259DA95U, 0xBE832DD4U, 0x7421D358U, 0xE0692949U, 0xC9C8448EU,
0xC2896A75U, 0x8E7978F4U, 0x583E6B99U, 0xB971DD27U, 0xE14FB6BEU, 0x88AD17F0U, 0x20AC66C9U, 0xCE3AB47DU,
0xDF4A1863U, 0x1A3182E5U, 0x51336097U, 0x537F4562U, 0x6477E0B1U, 0x6BAE84BBU, 0x81A01CFEU, 0x082B94F9U,
0x48685870U, 0x45FD198FU, 0xDE6C8794U, 0x7BF8B752U, 0x73D323ABU, 0x4B02E272U, 0x1F8F57E3U, 0x55AB2A66U,
0xEB2807B2U, 0xB5C2032FU, 0xC57B9A86U, 0x3708A5D3U, 0x2887F230U, 0xBFA5B223U, 0x036ABA02U, 0x16825CEDU,
0xCF1C2B8AU, 0x79B492A7U, 0x07F2F0F3U, 0x69E2A14EU, 0xDAF4CD65U, 0x05BED506U, 0x34621FD1U, 0xA6FE8AC4U,
0x2E539D34U, 0xF355A0A2U, 0x8AE13205U, 0xF6EB75A4U, 0x83EC390BU, 0x60EFAA40U, 0x719F065EU, 0x6E1051BDU,
0x218AF93EU, 0xDD063D96U, 0x3E05AEDDU, 0xE6BD464DU, 0x548DB591U, 0xC45D0571U, 0x06D46F04U, 0x5015FF60U,
0x98FB2419U, 0xBDE997D6U, 0x4043CC89U, 0xD99E7767U, 0xE842BDB0U, 0x898B8807U, 0x195B38E7U, 0xC8EEDB79U,
0x7C0A47A1U, 0x420FE97CU, 0x841EC9F8U, 0x00000000U, 0x80868309U, 0x2BED4832U, 0x1170AC1EU, 0x5A724E6CU,
0x0EFFFBFDU, 0x8538560FU, 0xAED51E3DU, 0x2D392736U, 0x0FD9640AU, 0x5CA62168U, 0x5B54D19BU, 0x362E3A24U,
0x0A67B10CU, 0x57E70F93U, 0xEE96D2B4U, 0x9B919E1BU, 0xC0C54F80U, 0xDC20A261U, 0x774B695AU, 0x121A161CU,
0x93BA0AE2U, 0xA02AE5C0U, 0x22E0433CU, 0x1B171D12U, 0x090D0B0EU, 0x8BC7ADF2U, 0xB6A8B92DU, 0x1EA9C814U,
0xF1198557U, 0x75074CAFU, 0x99DDBBEEU, 0x7F60FDA3U, 0x01269FF7U, 0x72F5BC5CU, 0x663BC544U, 0xFB7E345BU,
0x4329768BU, 0x23C6DCCBU, 0xEDFC68B6U, 0xE4F163B8U, 0x31DCCAD7U, 0x63851042U, 0x97224013U, 0xC6112084U,
0x4A247D85U, 0xBB3DF8D2U, 0xF93211AEU, 0x29A16DC7U, 0x9E2F4B1DU, 0xB230F3DCU, 0x8652EC0DU, 0xC1E3D077U,
0xB3166C2BU, 0x70B999A9U, 0x9448FA11U, 0xE9642247U, 0xFC8CC4A8U, 0xF03F1AA0U, 0x7D2CD856U, 0x3390EF22U,
0x494EC787U, 0x38D1C1D9U, 0xCAA2FE8CU, 0xD40B3698U, 0xF581CFA6U, 0x7ADE28A5U, 0xB78E26DAU, 0xADBFA43FU,
0x3A9DE42CU, 0x78920D50U, 0x5FCC9B6AU, 0x7E466254U, 0x8D13C2F6U, 0xD8B8E890U, 0x39F75E2EU, 0xC3AFF582U,
0x5D80BE9FU, 0xD0937C69U, 0xD52DA96FU, 0x2512B3CFU, 0xAC993BC8U, 0x187DA710U, 0x9C636EE8U, 0x3BBB7BDBU,
0x267809CDU, 0x5918F46EU, 0x9AB701ECU, 0x4F9AA883U, 0x956E65E6U, 0xFFE67EAAU, 0xBCCF0821U, 0x15E8E6EFU,
0xE79BD9BAU, 0x6F36CE4AU, 0x9F09D4EAU, 0xB07CD629U, 0xA4B2AF31U, 0x3F23312AU, 0xA59430C6U, 0xA266C035U,
0x4EBC3774U, 0x82CAA6FCU, 0x90D0B0E0U, 0xA7D81533U, 0x04984AF1U, 0xECDAF741U, 0xCD500E7FU, 0x91F62F17U,
0x4DD68D76U, 0xEFB04D43U, 0xAA4D54CCU, 0x9604DFE4U, 0xD1B5E39EU, 0x6A881B4CU, 0x2C1FB8C1U, 0x65517F46U,
0x5EEA049DU, 0x8C355D01U, 0x877473FAU, 0x0B412EFBU, 0x671D5AB3U, 0xDBD25292U, 0x105633E9U, 0xD647136DU,
0xD7618C9AU, 0xA10C7A37U, 0xF8148E59U, 0x133C89EBU, 0xA927EECEU, 0x61C935B7U, 0x1CE5EDE1U, 0x47B13C7AU,
0xD2DF599CU, 0xF2733F55U, 0x14CE7918U, 0xC737BF73U, 0xF7CDEA53U, 0xFDAA5B5FU, 0x3D6F14DFU, 0x44DB8678U,
0xAFF381CAU, 0x68C43EB9U, 0x24342C38U, 0xA3405FC2U, 0x1DC37216U, 0xE2250CBCU, 0x3C498B28U, 0x0D9541FFU,
0xA8017139U, 0x0CB3DE08U, 0xB4E49CD8U, 0x56C19064U, 0xCB84617BU, 0x32B670D5U, 0x6C5C7448U, 0xB85742D0U
};
const uint32_t AES::Td4[256] = {
0x52525252U, 0x09090909U, 0x6A6A6A6AU, 0xD5D5D5D5U, 0x30303030U, 0x36363636U, 0xA5A5A5A5U, 0x38383838U,
0xBFBFBFBFU, 0x40404040U, 0xA3A3A3A3U, 0x9E9E9E9EU, 0x81818181U, 0xF3F3F3F3U, 0xD7D7D7D7U, 0xFBFBFBFBU,
0x7C7C7C7CU, 0xE3E3E3E3U, 0x39393939U, 0x82828282U, 0x9B9B9B9BU, 0x2F2F2F2FU, 0xFFFFFFFFU, 0x87878787U,
0x34343434U, 0x8E8E8E8EU, 0x43434343U, 0x44444444U, 0xC4C4C4C4U, 0xDEDEDEDEU, 0xE9E9E9E9U, 0xCBCBCBCBU,
0x54545454U, 0x7B7B7B7BU, 0x94949494U, 0x32323232U, 0xA6A6A6A6U, 0xC2C2C2C2U, 0x23232323U, 0x3D3D3D3DU,
0xEEEEEEEEU, 0x4C4C4C4CU, 0x95959595U, 0x0B0B0B0BU, 0x42424242U, 0xFAFAFAFAU, 0xC3C3C3C3U, 0x4E4E4E4EU,
0x08080808U, 0x2E2E2E2EU, 0xA1A1A1A1U, 0x66666666U, 0x28282828U, 0xD9D9D9D9U, 0x24242424U, 0xB2B2B2B2U,
0x76767676U, 0x5B5B5B5BU, 0xA2A2A2A2U, 0x49494949U, 0x6D6D6D6DU, 0x8B8B8B8BU, 0xD1D1D1D1U, 0x25252525U,
0x72727272U, 0xF8F8F8F8U, 0xF6F6F6F6U, 0x64646464U, 0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U,
0xD4D4D4D4U, 0xA4A4A4A4U, 0x5C5C5C5CU, 0xCCCCCCCCU, 0x5D5D5D5DU, 0x65656565U, 0xB6B6B6B6U, 0x92929292U,
0x6C6C6C6CU, 0x70707070U, 0x48484848U, 0x50505050U, 0xFDFDFDFDU, 0xEDEDEDEDU, 0xB9B9B9B9U, 0xDADADADAU,
0x5E5E5E5EU, 0x15151515U, 0x46464646U, 0x57575757U, 0xA7A7A7A7U, 0x8D8D8D8DU, 0x9D9D9D9DU, 0x84848484U,
0x90909090U, 0xD8D8D8D8U, 0xABABABABU, 0x00000000U, 0x8C8C8C8CU, 0xBCBCBCBCU, 0xD3D3D3D3U, 0x0A0A0A0AU,
0xF7F7F7F7U, 0xE4E4E4E4U, 0x58585858U, 0x05050505U, 0xB8B8B8B8U, 0xB3B3B3B3U, 0x45454545U, 0x06060606U,
0xD0D0D0D0U, 0x2C2C2C2CU, 0x1E1E1E1EU, 0x8F8F8F8FU, 0xCACACACAU, 0x3F3F3F3FU, 0x0F0F0F0FU, 0x02020202U,
0xC1C1C1C1U, 0xAFAFAFAFU, 0xBDBDBDBDU, 0x03030303U, 0x01010101U, 0x13131313U, 0x8A8A8A8AU, 0x6B6B6B6BU,
0x3A3A3A3AU, 0x91919191U, 0x11111111U, 0x41414141U, 0x4F4F4F4FU, 0x67676767U, 0xDCDCDCDCU, 0xEAEAEAEAU,
0x97979797U, 0xF2F2F2F2U, 0xCFCFCFCFU, 0xCECECECEU, 0xF0F0F0F0U, 0xB4B4B4B4U, 0xE6E6E6E6U, 0x73737373U,
0x96969696U, 0xACACACACU, 0x74747474U, 0x22222222U, 0xE7E7E7E7U, 0xADADADADU, 0x35353535U, 0x85858585U,
0xE2E2E2E2U, 0xF9F9F9F9U, 0x37373737U, 0xE8E8E8E8U, 0x1C1C1C1CU, 0x75757575U, 0xDFDFDFDFU, 0x6E6E6E6EU,
0x47474747U, 0xF1F1F1F1U, 0x1A1A1A1AU, 0x71717171U, 0x1D1D1D1DU, 0x29292929U, 0xC5C5C5C5U, 0x89898989U,
0x6F6F6F6FU, 0xB7B7B7B7U, 0x62626262U, 0x0E0E0E0EU, 0xAAAAAAAAU, 0x18181818U, 0xBEBEBEBEU, 0x1B1B1B1BU,
0xFCFCFCFCU, 0x56565656U, 0x3E3E3E3EU, 0x4B4B4B4BU, 0xC6C6C6C6U, 0xD2D2D2D2U, 0x79797979U, 0x20202020U,
0x9A9A9A9AU, 0xDBDBDBDBU, 0xC0C0C0C0U, 0xFEFEFEFEU, 0x78787878U, 0xCDCDCDCDU, 0x5A5A5A5AU, 0xF4F4F4F4U,
0x1F1F1F1FU, 0xDDDDDDDDU, 0xA8A8A8A8U, 0x33333333U, 0x88888888U, 0x07070707U, 0xC7C7C7C7U, 0x31313131U,
0xB1B1B1B1U, 0x12121212U, 0x10101010U, 0x59595959U, 0x27272727U, 0x80808080U, 0xECECECECU, 0x5F5F5F5FU,
0x60606060U, 0x51515151U, 0x7F7F7F7FU, 0xA9A9A9A9U, 0x19191919U, 0xB5B5B5B5U, 0x4A4A4A4AU, 0x0D0D0D0DU,
0x2D2D2D2DU, 0xE5E5E5E5U, 0x7A7A7A7AU, 0x9F9F9F9FU, 0x93939393U, 0xC9C9C9C9U, 0x9C9C9C9CU, 0xEFEFEFEFU,
0xA0A0A0A0U, 0xE0E0E0E0U, 0x3B3B3B3BU, 0x4D4D4D4DU, 0xAEAEAEAEU, 0x2A2A2A2AU, 0xF5F5F5F5U, 0xB0B0B0B0U,
0xC8C8C8C8U, 0xEBEBEBEBU, 0xBBBBBBBBU, 0x3C3C3C3CU, 0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U,
0x17171717U, 0x2B2B2B2BU, 0x04040404U, 0x7E7E7E7EU, 0xBABABABAU, 0x77777777U, 0xD6D6D6D6U, 0x26262626U,
0xE1E1E1E1U, 0x69696969U, 0x14141414U, 0x63636363U, 0x55555555U, 0x21212121U, 0x0C0C0C0CU, 0x7D7D7D7DU
};
const uint32_t AES::rcon[11] = {
0x00000000U, 0x01000000U, 0x02000000U, 0x04000000U, 0x08000000U, 0x10000000U, 0x20000000U, 0x40000000U,
0x80000000U, 0x1B000000U, 0x36000000U
};
AES::AES()
{
Nb = 4;
Nk = 4;
Nr = 10;
std::fill_n(_iv, 16, 0);
std::fill_n(_erk, 60, 0);
std::fill_n(_drk, 60, 0);
_tp = 0;
}
AES::~AES()
{
CleanUp();
}
void AES::CleanUp()
{
std::fill_n(_iv, 16, 0);
std::fill_n(_erk, 60, 0);
std::fill_n(_drk, 60, 0);
_tp = 0;
}
void AES::SetKey(const uint8_t *key, Mode mode)
{
int i;
uint32_t tmp;
int ks_sz;
switch (mode) {
case AES_128: Nk = 4; Nr = 10; break;
case AES_192: Nk = 6; Nr = 12; break;
case AES_256: Nk = 8; Nr = 14; break;
}
ks_sz = Nb * (Nr + 1);
for (i = 0; i < Nk; i++) {
_erk[i] = (key[4*i] << 24) | (key[4*i+1] << 16) | (key[4*i+2] << 8) | key[4*i+3];
}
for (; i < ks_sz; i++) {
tmp = _erk[i-1];
if (i % Nk == 0) {
tmp = (Te4[(tmp >> 16) & 0xFF] & 0xFF000000) ^ (Te4[(tmp >> 8) & 0xFF] & 0xFF0000) ^ (Te4[tmp & 0xFF] & 0xFF00) ^ (Te4[(tmp >> 24) & 0xFF] & 0xFF) ^ rcon[i / Nk];
} else if ((Nk > 6) && ((i % Nk) == 4)) {
tmp = (Te4[(tmp >> 24) & 0xFF] & 0xFF000000) ^ (Te4[(tmp >> 16) & 0xFF] & 0xFF0000) ^ (Te4[(tmp >> 8) & 0xFF] & 0xFF00) ^ (Te4[tmp & 0xFF] & 0xFF);
}
_erk[i] = _erk[i-Nk] ^ tmp;
}
for (i = 0; i < ks_sz; i += 4) {
_drk[i ] = _erk[ks_sz - i - 4];
_drk[i + 1] = _erk[ks_sz - i - 3];
_drk[i + 2] = _erk[ks_sz - i - 2];
_drk[i + 3] = _erk[ks_sz - i - 1];
}
for (i = 4; i < (ks_sz - 4); i++) {
_drk[i] = Td0[Te4[(_drk[i] >> 24)] & 0xFF] ^ Td1[Te4[(_drk[i] >> 16) & 0xFF] & 0xFF] ^ Td2[Te4[(_drk[i] >> 8) & 0xFF] & 0xFF] ^ Td3[Te4[_drk[i] & 0xFF] & 0xFF];
}
SetIV(0);
}
void AES::Encrypt(const uint8_t *src, uint8_t *dst)
{
uint32_t s0, s1, s2, s3, t0, t1, t2, t3;
int r, ki;
s0 = (src[0] << 24 | src[1] << 16 | src[2] << 8 | src[3]) ^ _erk[0];
s1 = (src[4] << 24 | src[5] << 16 | src[6] << 8 | src[7]) ^ _erk[1];
s2 = (src[8] << 24 | src[9] << 16 | src[10] << 8 | src[11]) ^ _erk[2];
s3 = (src[12] << 24 | src[13] << 16 | src[14] << 8 | src[15]) ^ _erk[3];
ki = 4;
r = Nr >> 1;
for (;;) {
t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xFF] ^ Te2[(s2 >> 8) & 0xFF] ^ Te3[s3 & 0xFF] ^ _erk[ki++];
t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xFF] ^ Te2[(s3 >> 8) & 0xFF] ^ Te3[s0 & 0xFF] ^ _erk[ki++];
t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xFF] ^ Te2[(s0 >> 8) & 0xFF] ^ Te3[s1 & 0xFF] ^ _erk[ki++];
t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xFF] ^ Te2[(s1 >> 8) & 0xFF] ^ Te3[s2 & 0xFF] ^ _erk[ki++];
if (--r == 0) break;
s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xFF] ^ Te2[(t2 >> 8) & 0xFF] ^ Te3[t3 & 0xFF] ^ _erk[ki++];
s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xFF] ^ Te2[(t3 >> 8) & 0xFF] ^ Te3[t0 & 0xFF] ^ _erk[ki++];
s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xFF] ^ Te2[(t0 >> 8) & 0xFF] ^ Te3[t1 & 0xFF] ^ _erk[ki++];
s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xFF] ^ Te2[(t1 >> 8) & 0xFF] ^ Te3[t2 & 0xFF] ^ _erk[ki++];
}
s0 = (Te4[t0 >> 24] & 0xFF000000) ^ (Te4[(t1 >> 16) & 0xFF] & 0xFF0000) ^ (Te4[(t2 >> 8) & 0xFF] & 0xFF00) ^ (Te4[t3 & 0xFF] & 0xFF) ^ _erk[ki++];
s1 = (Te4[t1 >> 24] & 0xFF000000) ^ (Te4[(t2 >> 16) & 0xFF] & 0xFF0000) ^ (Te4[(t3 >> 8) & 0xFF] & 0xFF00) ^ (Te4[t0 & 0xFF] & 0xFF) ^ _erk[ki++];
s2 = (Te4[t2 >> 24] & 0xFF000000) ^ (Te4[(t3 >> 16) & 0xFF] & 0xFF0000) ^ (Te4[(t0 >> 8) & 0xFF] & 0xFF00) ^ (Te4[t1 & 0xFF] & 0xFF) ^ _erk[ki++];
s3 = (Te4[t3 >> 24] & 0xFF000000) ^ (Te4[(t0 >> 16) & 0xFF] & 0xFF0000) ^ (Te4[(t1 >> 8) & 0xFF] & 0xFF00) ^ (Te4[t2 & 0xFF] & 0xFF) ^ _erk[ki++];
dst[0] = s0 >> 24; dst[1] = s0 >> 16; dst[2] = s0 >> 8; dst[3] = s0;
dst[4] = s1 >> 24; dst[5] = s1 >> 16; dst[6] = s1 >> 8; dst[7] = s1;
dst[8] = s2 >> 24; dst[9] = s2 >> 16; dst[10] = s2 >> 8; dst[11] = s2;
dst[12] = s3 >> 24; dst[13] = s3 >> 16; dst[14] = s3 >> 8; dst[15] = s3;
}
void AES::Decrypt(const uint8_t *src, uint8_t *dst)
{
uint32_t s0, s1, s2, s3, t0, t1, t2, t3;
int r, ki;
s0 = (src[0] << 24 | src[1] << 16 | src[2] << 8 | src[3]) ^ _drk[0];
s1 = (src[4] << 24 | src[5] << 16 | src[6] << 8 | src[7]) ^ _drk[1];
s2 = (src[8] << 24 | src[9] << 16 | src[10] << 8 | src[11]) ^ _drk[2];
s3 = (src[12] << 24 | src[13] << 16 | src[14] << 8 | src[15]) ^ _drk[3];
ki = 4;
r = Nr >> 1;
for (;;) {
t0 = Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xFF] ^ Td2[(s2 >> 8) & 0xFF] ^ Td3[s1 & 0xFF] ^ _drk[ki++];
t1 = Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xFF] ^ Td2[(s3 >> 8) & 0xFF] ^ Td3[s2 & 0xFF] ^ _drk[ki++];
t2 = Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xFF] ^ Td2[(s0 >> 8) & 0xFF] ^ Td3[s3 & 0xFF] ^ _drk[ki++];
t3 = Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xFF] ^ Td2[(s1 >> 8) & 0xFF] ^ Td3[s0 & 0xFF] ^ _drk[ki++];
if (--r == 0) break;
s0 = Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xFF] ^ Td2[(t2 >> 8) & 0xFF] ^ Td3[t1 & 0xFF] ^ _drk[ki++];
s1 = Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xFF] ^ Td2[(t3 >> 8) & 0xFF] ^ Td3[t2 & 0xFF] ^ _drk[ki++];
s2 = Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xFF] ^ Td2[(t0 >> 8) & 0xFF] ^ Td3[t3 & 0xFF] ^ _drk[ki++];
s3 = Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xFF] ^ Td2[(t1 >> 8) & 0xFF] ^ Td3[t0 & 0xFF] ^ _drk[ki++];
}
s0 = (Td4[t0 >> 24] & 0xFF000000) ^ (Td4[(t3 >> 16) & 0xFF] & 0xFF0000) ^ (Td4[(t2 >> 8) & 0xFF] & 0xFF00) ^ (Td4[t1 & 0xFF] & 0xFF) ^ _drk[ki++];
s1 = (Td4[t1 >> 24] & 0xFF000000) ^ (Td4[(t0 >> 16) & 0xFF] & 0xFF0000) ^ (Td4[(t3 >> 8) & 0xFF] & 0xFF00) ^ (Td4[t2 & 0xFF] & 0xFF) ^ _drk[ki++];
s2 = (Td4[t2 >> 24] & 0xFF000000) ^ (Td4[(t1 >> 16) & 0xFF] & 0xFF0000) ^ (Td4[(t0 >> 8) & 0xFF] & 0xFF00) ^ (Td4[t3 & 0xFF] & 0xFF) ^ _drk[ki++];
s3 = (Td4[t3 >> 24] & 0xFF000000) ^ (Td4[(t2 >> 16) & 0xFF] & 0xFF0000) ^ (Td4[(t1 >> 8) & 0xFF] & 0xFF00) ^ (Td4[t0 & 0xFF] & 0xFF) ^ _drk[ki++];
dst[0] = s0 >> 24; dst[1] = s0 >> 16; dst[2] = s0 >> 8; dst[3] = s0;
dst[4] = s1 >> 24; dst[5] = s1 >> 16; dst[6] = s1 >> 8; dst[7] = s1;
dst[8] = s2 >> 24; dst[9] = s2 >> 16; dst[10] = s2 >> 8; dst[11] = s2;
dst[12] = s3 >> 24; dst[13] = s3 >> 16; dst[14] = s3 >> 8; dst[15] = s3;
}
void AES::SetIV(const uint8_t *iv)
{
size_t j;
_tp = 0;
if (iv == 0)
iv = zeros;
for (j = 0; j < 16; j++) _iv[j] = iv[j];
}
void AES::EncryptCBC(const uint8_t *src, uint8_t *dst, size_t size)
{
size_t i, j;
uint8_t tmp[16];
for (i = 0; i < size; i += 16) {
for (j = 0; j < 16; j++) tmp[j] = src[i+j] ^ _iv[j];
Encrypt(tmp, &dst[i]);
for (j = 0; j < 16; j++) _iv[j] = dst[i+j];
}
}
void AES::DecryptCBC(const uint8_t *src, uint8_t *dst, size_t size)
{
size_t i, j;
uint8_t tmp[16];
for (i = 0; i < size; i += 16) {
for (j = 0; j < 16; j++) tmp[j] = src[i+j];
Decrypt(&src[i], &dst[i]);
for (j = 0; j < 16; j++) {
dst[i+j] ^= _iv[j];
_iv[j] = tmp[j];
}
}
}
void AES::EncryptCFB(const uint8_t *src, uint8_t *dst, size_t size)
{
size_t i;
for (i = 0; i < size; i++, _tp = (_tp + 1) & 15) {
if (_tp == 0)
Encrypt(_iv, _iv);
dst[i] = src[i] ^ _iv[_tp];
_iv[_tp] = dst[i];
}
}
void AES::DecryptCFB(const uint8_t *src, uint8_t *dst, size_t size)
{
size_t i;
uint8_t t;
for (i = 0; i < size; i++, _tp = (_tp + 1) & 15) {
if (_tp == 0)
Encrypt(_iv, _iv);
t = src[i];
dst[i] = src[i] ^ _iv[_tp];
_iv[_tp] = t;
}
}
void AES::CryptOFB(const uint8_t *src, uint8_t *dst, size_t size)
{
size_t i;
for (i = 0; i < size; i++, _tp = (_tp + 1) & 15) {
if (_tp == 0)
Encrypt(_iv, _iv);
dst[i] = src[i] ^ _iv[_tp];
}
}
const uint8_t AES::zeros[32] = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
+172
查看文件
@@ -0,0 +1,172 @@
#pragma once
#include <cstddef>
#include <cstdint>
/**
* @brief The AES class
*
* Class for AES en- and decryption. Supports all AES modes specified
* in NIST FIPS-197.
*
* Several chaining modes are supported, namely ECB, CBC, CFB and OFB.
*
* Usage:
*
* @li Set a key with SetKey.
* @li If necessary, set an IV with SetIV.
* @li Encrypt / Decrypt your data with the desired functions.
* @li When you are done, CleanUp behind you to make life harder for hackers.
*/
class AES
{
public:
AES();
~AES();
/**
* @brief Encryption key size in bits.
*/
enum Mode {
AES_128,
AES_192,
AES_256
};
/**
* @brief Clean Up
*
* Deletes all data which may contain key / plaintext material.
*
* @note After using this, a new key must be set before additional
* encryption is done.
*/
void CleanUp();
/**
* @brief Set AES Key
*
* Sets the AES encryption key for this instance. Also sets the IV
* to 0.
*
* @param key Key data. Length depends on the mode parameter.
* @param mode Encryption mode (128, 192 or 256 bit key).
*/
void SetKey(const uint8_t *key, Mode mode);
/**
* @brief Set Initialization Vector
*
* Set the initialization vector. This also resets the counter for the
* stream cipher modes (CFB, OFB).
*
* @param iv Initialization Vector. If NULL, a 0-vector is used.
*/
void SetIV(const uint8_t *iv);
/**
* @brief Encrypt Block
*
* Encrypts a block of 16 bytes with the previously selected key (ECB mode).
*
* @param src Plaintext block (128 bits, 16 bytes)
* @param dst Encrypted block (128 bits, 16 bytes)
*/
void Encrypt(const uint8_t *src, uint8_t *dst);
/**
* @brief Decrypt Block
*
* Decrypts a block of 16 bytes with the previously selected key (ECB mode).
*
* @param src Encrypted block (128 bits, 16 bytes)
* @param dst Decrypted block (128 bits, 16 bytes)
*/
void Decrypt(const uint8_t *src, uint8_t *dst);
/**
* @brief Encrypt CBC
*
* Encrypt data in CBC mode.
*
* @param src Plaintext data.
* @param dst Encrypted data.
* @param size Number of bytes. Must be a multiple of 16.
*/
void EncryptCBC(const uint8_t *src, uint8_t *dst, size_t size);
/**
* @brief Decrypt CBC
*
* Decrypt data in CBC mode.
*
* @param src Encrypted data.
* @param dst Decrypted data.
* @param size Number of bytes. Must be a multiple of 16.
*/
void DecryptCBC(const uint8_t *src, uint8_t *dst, size_t size);
/**
* @brief Encrypt CFB
*
* Encrypt data in CFB mode.
*
* @param src Plaintext data.
* @param dst Encrypted data.
* @param size Number of bytes.
*/
void EncryptCFB(const uint8_t *src, uint8_t *dst, size_t size);
/**
* @brief Decrypt CFB
*
* Decrypt data in CFB mode.
*
* @param src Encrypted data.
* @param dst Decrypted data.
* @param size Number of bytes.
*/
void DecryptCFB(const uint8_t *src, uint8_t *dst, size_t size);
/**
* @brief Crypt OFB
*
* En- / Decrypt data in OFB mode.
*
* @param src Input data.
* @param dst Output data.
* @param size Number of bytes.
*/
void CryptOFB(const uint8_t *src, uint8_t *dst, size_t size);
private:
/// Encryption round key.
uint32_t _erk[60];
/// Decryption round key.
uint32_t _drk[60];
/// Initialization vector.
uint8_t _iv[16];
/// Byte counter for CFB / OFB modes.
int _tp;
// AES constants (depending on mode)
int Nk;
int Nr;
int Nb;
// AES tables
static const uint32_t Td0[256];
static const uint32_t Td1[256];
static const uint32_t Td2[256];
static const uint32_t Td3[256];
static const uint32_t Td4[256];
static const uint32_t Te0[256];
static const uint32_t Te1[256];
static const uint32_t Te2[256];
static const uint32_t Te3[256];
static const uint32_t Te4[256];
static const uint32_t rcon[11];
/// Zero data
static const uint8_t zeros[32];
};
+102
查看文件
@@ -0,0 +1,102 @@
#include <ApfsLib/Endian.h>
#include "AesXts.h"
#include <algorithm>
#include <iterator>
AesXts::AesXts()
{
CleanUp();
}
AesXts::~AesXts()
{
}
void AesXts::CleanUp()
{
m_aes_1.CleanUp();
m_aes_2.CleanUp();
}
void AesXts::SetKey(const uint8_t* key1, const uint8_t* key2)
{
m_aes_1.SetKey(key1, AES::AES_128);
m_aes_2.SetKey(key2, AES::AES_128);
}
void AesXts::Encrypt(uint8_t* cipher, const uint8_t* plain, std::size_t size, uint64_t unit_no)
{
uint8_t pp[0x10];
uint8_t cc[0x10];
uint64_t tweak[2];
size_t k = 0;
tweak[0] = htole64(unit_no);
tweak[1] = 0;
m_aes_2.Encrypt(reinterpret_cast<const uint8_t *>(tweak), reinterpret_cast<uint8_t *>(tweak));
for (k = 0; k < size; k += 0x10)
{
Xor128(pp, plain + k, tweak);
m_aes_1.Encrypt(pp, cc);
Xor128(cipher + k, cc, tweak);
MultiplyTweak(tweak);
}
}
void AesXts::Decrypt(uint8_t* plain, const uint8_t* cipher, std::size_t size, uint64_t unit_no)
{
uint8_t pp[0x10];
uint8_t cc[0x10];
uint64_t tweak[2];
size_t k = 0;
tweak[0] = htole64(unit_no);
tweak[1] = 0;
m_aes_2.Encrypt(reinterpret_cast<const uint8_t *>(tweak), reinterpret_cast<uint8_t *>(tweak));
for (k = 0; k < size; k += 0x10)
{
Xor128(cc, cipher + k, tweak);
m_aes_1.Decrypt(cc, pp);
Xor128(plain + k, pp, tweak);
MultiplyTweak(tweak);
}
}
void AesXts::Xor128(void *out, const void *op1, const void *op2)
{
reinterpret_cast<uint64_t*>(out)[0] = reinterpret_cast<const uint64_t*>(op1)[0] ^ reinterpret_cast<const uint64_t*>(op2)[0];
reinterpret_cast<uint64_t*>(out)[1] = reinterpret_cast<const uint64_t*>(op1)[1] ^ reinterpret_cast<const uint64_t*>(op2)[1];
}
void AesXts::MultiplyTweak(uint64_t* tweak)
{
uint8_t c1;
uint8_t c2;
#if (__BYTE_ORDER == __LITTLE_ENDIAN)
c1 = (tweak[0] & 0x8000000000000000ULL) ? 1 : 0;
c2 = (tweak[1] & 0x8000000000000000ULL) ? 0x87 : 0;
tweak[0] = (tweak[0] << 1) ^ c2;
tweak[1] = (tweak[1] << 1) | c1;
#else
uint64_t t0;
uint64_t t1;
t0 = le64toh(tweak[0]);
t1 = le64toh(tweak[1]);
c1 = (t0 & 0x8000000000000000ULL) ? 1 : 0;
c2 = (t1 & 0x8000000000000000ULL) ? 0x87 : 0;
t0 = (t0 << 1) ^ c2;
t1 = (t1 << 1) | c1;
tweak[0] = htole64(t0);
tweak[1] = htole64(t1);
#endif
}
+26
查看文件
@@ -0,0 +1,26 @@
#pragma once
#include <cstdint>
#include "Aes.h"
class AesXts
{
public:
AesXts();
~AesXts();
void CleanUp();
void SetKey(const uint8_t *key1, const uint8_t *key2);
void Encrypt(uint8_t *cipher, const uint8_t *plain, size_t size, uint64_t unit_no);
void Decrypt(uint8_t *plain, const uint8_t *cipher, size_t size, uint64_t unit_no);
private:
void Xor128(void *out, const void *op1, const void *op2);
void MultiplyTweak(uint64_t *tweak);
AES m_aes_1;
AES m_aes_2;
};
+184
查看文件
@@ -0,0 +1,184 @@
#include <cstdio>
#include <cinttypes>
#include <cstring>
#include "Asn1Der.h"
const uint8_t* der_decode_tag(der_tag_t& tag, const uint8_t* der, const uint8_t* der_end)
{
uint64_t t;
uint8_t b;
uint8_t flg;
tag = 0;
if (der == nullptr || der >= der_end)
return nullptr;
flg = *der++;
if ((flg & 0x1F) == 0x1F) {
t = 0;
do {
if (der >= der_end) return nullptr;
b = *der++;
t = t << 7 | (b & 0x7F);
} while (b & 0x80);
tag = static_cast<uint64_t>(flg & 0xE0) << 56 | (t & 0x1FFFFFFFFFFFFFFFU);
}
else {
tag = static_cast<uint64_t>(flg & 0xE0) << 56 | (flg & 0x1F);
}
return der;
}
const uint8_t* der_decode_len(size_t& len, const uint8_t* der, const uint8_t* der_end)
{
uint64_t s;
uint8_t nb;
unsigned k;
len = 0;
if (der == nullptr || der >= der_end)
return nullptr;
nb = *der++;
if (nb & 0x80) {
s = 0;
nb &= 0x7F;
if ((der + nb) >= der_end) return nullptr;
for (k = 0; k < nb; k++)
s = s << 8 | *der++;
len = s;
}
else {
len = nb;
}
return der;
}
const uint8_t* der_decode_tl(der_tag_t expected_tag, size_t& len, const uint8_t* der, const uint8_t* der_end)
{
der_tag_t tag;
der = der_decode_tag(tag, der, der_end);
if (tag != expected_tag)
return nullptr;
der = der_decode_len(len, der, der_end);
if (der + len > der_end)
return nullptr;
return der;
}
const uint8_t* der_decode_constructed_tl(der_tag_t expected_tag, const uint8_t*& body_end, const uint8_t* der, const uint8_t* der_end)
{
size_t len;
der = der_decode_tl(expected_tag, len, der, der_end);
if (der == nullptr)
return nullptr;
body_end = der + len;
return der;
}
const uint8_t* der_decode_sequence_tl(const uint8_t*& body_end, const uint8_t* der, const uint8_t* der_end)
{
return der_decode_constructed_tl(DER_CONSTRUCTED | DER_SEQUENCE, body_end, der, der_end);
}
const uint8_t* der_decode_uint(size_t n, uint64_t& val, const uint8_t* der, const uint8_t* der_end)
{
uint64_t v = 0;
size_t k;
val = 0;
if (der == nullptr || (der + n) > der_end)
return nullptr;
for (k = 0; k < n; k++)
v = v << 8 | *der++;
val = v;
return der;
}
const uint8_t * der_decode_uint64(der_tag_t expected_tag, uint64_t& val, const uint8_t* der, const uint8_t* der_end)
{
size_t len;
val = 0;
der = der_decode_tl(expected_tag, len, der, der_end);
if (der == nullptr || len > 8) return nullptr;
der = der_decode_uint(len, val, der, der_end);
return der;
}
const uint8_t * der_decode_octet_string_copy(der_tag_t expected_tag, uint8_t* buf, size_t len, const uint8_t* der, const uint8_t* der_end)
{
size_t slen;
der = der_decode_tl(expected_tag, slen, der, der_end);
if (der == nullptr || slen != len || (der + slen) > der_end) return nullptr;
memcpy(buf, der, slen);
return der + slen;
}
/*
static void der_dump_data(const uint8_t* data, size_t len)
{
size_t y;
size_t x;
constexpr size_t ls = 32;
size_t ll;
for (y = 0; y < len; y += ls) {
ll = ls;
if ((len - y) < ls) ll = len - y;
for (x = 0; x < (ll - 1); x++)
printf("%02X ", data[y + x]);
printf("%02X\n", data[y + x]);
}
}
*/
static void der_dump_hex(const uint8_t* data, size_t len)
{
size_t n;
for (n = 0; n < len; n++)
printf("%02X", data[n]);
printf("\n");
}
static void der_dump_internal(const uint8_t* der, const uint8_t* der_end, int indent)
{
der_tag_t tag;
size_t len;
int n;
// const uint8_t* der_start = der;
// printf("DER %zX\n", der_end - der);
while (der < der_end && der != nullptr) {
// printf("[%04zX", der - der_start);
der = der_decode_tag(tag, der, der_end);
der = der_decode_len(len, der, der_end);
// printf("/%04zX] ", der - der_start);
if (der == nullptr || (der + len) > der_end) break;
for (n = 0; n < indent; n++)
putchar(' ');
printf("%016" PRIX64 " %04zX", tag, len);
if (tag & DER_CONSTRUCTED) {
printf("\n");
der_dump_internal(der, der + len, indent + 2);
} else {
printf(" : ");
der_dump_hex(der, len);
}
der += len;
}
if (der == nullptr)
printf("Malformed ASN.1 DER\n");
}
void der_dump(const uint8_t* data, size_t size)
{
// der_dump_data(data, size);
der_dump_internal(data, data + size, 0);
}
+53
查看文件
@@ -0,0 +1,53 @@
#pragma once
#include <cstdint>
#include <cstddef>
typedef uint64_t der_tag_t;
static constexpr der_tag_t DER_CONSTRUCTED = 0x2000000000000000U;
static constexpr der_tag_t DER_CONTEXT_SPECIFIC = 0x8000000000000000U;
enum der_tag_type_t
{
DER_BOOLEAN = 1,
DER_INTEGER = 2,
DER_BIT_STRING = 3,
DER_OCTET_STRING = 4,
DER_NULL = 5,
DER_OBJECT_IDENTIFIER = 6,
DER_OBJECT_DESCRIPTOR = 7,
DER_EXTERNAL = 8,
DER_REAL = 9,
DER_ENUMERATED = 10,
DER_EMBEDDED_PDV = 11,
DER_UTF8String = 12,
DER_RELATIVE_OID = 13,
DER_SEQUENCE = 16,
DER_SET = 17,
DER_NumericString = 18,
DER_PrintableString = 19,
DER_TeletexString = 20,
DER_VideotexString = 21,
DER_IA5String = 22,
DER_UTCTime = 23,
DER_GeneralizedTime = 24,
DER_GraphicString = 25,
DER_VisibleString = 26,
DER_GeneralString = 27,
DER_UniversalString = 28,
DER_CHARACTER_STRING = 29,
DER_BMPString = 30
};
const uint8_t* der_decode_tag(der_tag_t& tag, const uint8_t* der, const uint8_t* der_end);
const uint8_t* der_decode_len(size_t& len, const uint8_t* der, const uint8_t* der_end);
const uint8_t* der_decode_tl(der_tag_t expected_tag, size_t& len, const uint8_t* der, const uint8_t* der_end);
const uint8_t* der_decode_constructed_tl(der_tag_t expected_tag, const uint8_t*& body_end, const uint8_t* der, const uint8_t* der_end);
const uint8_t* der_decode_sequence_tl(const uint8_t*& body_end, const uint8_t* der, const uint8_t* der_end);
const uint8_t* der_decode_uint(size_t n, uint64_t& val, const uint8_t* der, const uint8_t* der_end);
const uint8_t* der_decode_uint64(der_tag_t expected_tag, uint64_t& val, const uint8_t* der, const uint8_t* der_end);
const uint8_t* der_decode_octet_string_copy(der_tag_t expected_tag, uint8_t* buf, size_t len, const uint8_t* der, const uint8_t* der_end);
void der_dump(const uint8_t* data, size_t size);
+287
查看文件
@@ -0,0 +1,287 @@
#include "Sha1.h"
#include "Sha256.h"
#include "Crypto.h"
// #include "Util.h"
#include <cstring>
#include <cassert>
#include <iostream>
#include <iomanip>
#include <ApfsLib/Endian.h>
union Rfc3394_Unit {
uint64_t u64[2];
uint8_t u8[16];
};
constexpr uint64_t rfc_3394_default_iv = 0xA6A6A6A6A6A6A6A6ULL;
// TODO: Not tested on big-endian machines ...
void Rfc3394_KeyWrap(uint8_t *crypto, const uint8_t *plain, size_t size, const uint8_t *key, AES::Mode aes_mode, uint64_t iv)
{
Rfc3394_Unit u;
uint64_t a;
uint64_t r[6];
int i;
int j;
int n = size / sizeof(uint64_t);
uint64_t t;
const uint64_t *p = reinterpret_cast<const uint64_t *>(plain);
uint64_t *c = reinterpret_cast<uint64_t *>(crypto);
AES aes;
aes.SetKey(key, aes_mode);
a = iv;
t = 1;
for (i = 0; i < n; i++)
r[i] = p[i];
for (j = 0; j < 6; j++)
{
for (i = 0; i < n; i++)
{
u.u64[1] = r[i];
u.u64[0] = a;
aes.Encrypt(u.u8, u.u8);
a = u.u64[0] ^ bswap_be(t);
r[i] = u.u64[1];
t++;
}
}
c[0] = a;
for (i = 0; i < n; i++)
c[i + 1] = r[i];
}
// TODO: Not tested on big-endian machines ...
bool Rfc3394_KeyUnwrap(uint8_t *plain, const uint8_t *crypto, size_t size, const uint8_t *key, AES::Mode aes_mode, uint64_t *iv)
{
Rfc3394_Unit u;
uint64_t a;
uint64_t r[6];
int i;
int j;
int n = size / sizeof(uint64_t);
uint64_t t;
const uint64_t *c = reinterpret_cast<const uint64_t *>(crypto);
uint64_t *p = reinterpret_cast<uint64_t *>(plain);
AES aes;
aes.SetKey(key, aes_mode);
t = 6 * n;
a = c[0];
for (i = 0; i < n; i++)
r[i] = c[i + 1];
for (j = 5; j >= 0; j--)
{
for (i = n - 1; i >= 0; i--)
{
u.u64[0] = a ^ bswap_be(t);
u.u64[1] = r[i];
aes.Decrypt(u.u8, u.u8);
a = u.u64[0];
r[i] = u.u64[1];
t--;
}
}
for (i = 0; i < n; i++)
p[i] = r[i];
if (iv)
*iv = a;
return a == rfc_3394_default_iv;
}
void HMAC_SHA1(const uint8_t *key, size_t key_len, const uint8_t *data, size_t data_len, uint8_t *mac)
{
uint8_t kdata[0x40];
uint8_t digest[0x14];
constexpr uint8_t ipad = 0x36;
constexpr uint8_t opad = 0x5C;
Sha1 sha1;
if (key_len > sizeof(kdata))
{
sha1.Init();
sha1.Update(key, key_len);
sha1.Final(digest);
memcpy(kdata, digest, sizeof(digest));
key_len = sizeof(digest);
}
else
{
memcpy(kdata, key, key_len);
}
if (key_len < sizeof(kdata))
memset(kdata + key_len, 0, sizeof(kdata) - key_len);
for (size_t k = 0; k < sizeof(kdata); k++)
kdata[k] ^= ipad;
sha1.Init();
sha1.Update(kdata, sizeof(kdata));
sha1.Update(data, data_len);
sha1.Final(digest);
for (size_t k = 0; k < sizeof(kdata); k++)
kdata[k] ^= (ipad ^ opad);
sha1.Init();
sha1.Update(kdata, sizeof(kdata));
sha1.Update(digest, sizeof(digest));
sha1.Final(mac);
memset(digest, 0, sizeof(digest));
}
void HMAC_SHA256(const uint8_t *key, size_t key_len, const uint8_t *data, size_t data_len, uint8_t *mac)
{
uint8_t kdata[0x40];
uint8_t digest[0x20];
constexpr uint8_t ipad = 0x36;
constexpr uint8_t opad = 0x5C;
SHA256 sha256;
if (key_len > sizeof(kdata))
{
sha256.Init();
sha256.Update(key, key_len);
sha256.Final(digest);
memcpy(kdata, digest, sizeof(digest));
key_len = sizeof(digest);
}
else
{
memcpy(kdata, key, key_len);
}
if (key_len < sizeof(kdata))
memset(kdata + key_len, 0, sizeof(kdata) - key_len);
for (size_t k = 0; k < sizeof(kdata); k++)
kdata[k] ^= ipad;
sha256.Init();
sha256.Update(kdata, sizeof(kdata));
sha256.Update(data, data_len);
sha256.Final(digest);
for (size_t k = 0; k < sizeof(kdata); k++)
kdata[k] ^= (ipad ^ opad);
sha256.Init();
sha256.Update(kdata, sizeof(kdata));
sha256.Update(digest, sizeof(digest));
sha256.Final(mac);
memset(digest, 0, sizeof(digest));
}
void PBKDF2_HMAC_SHA1(const uint8_t* pw, size_t pw_len, const uint8_t* salt, size_t salt_len, int iterations, uint8_t* derived_key, size_t dk_len)
{
assert(salt_len <= 0x20);
assert(dk_len <= 0x20);
constexpr size_t h_len = 0x14;
size_t r;
size_t l;
uint8_t t[h_len];
uint8_t u[h_len];
uint8_t s[0x24];
size_t k;
int j;
uint32_t i;
size_t n;
r = dk_len % h_len;
l = dk_len / h_len;
if (r > 0) l++;
for (i = 1, k = 0; k < dk_len; i++, k += h_len)
{
// F(P,S,c,i)
memcpy(s, salt, salt_len);
s[salt_len + 0] = (i >> 24) & 0xFF;
s[salt_len + 1] = (i >> 16) & 0xFF;
s[salt_len + 2] = (i >> 8) & 0xFF;
s[salt_len + 3] = i & 0xFF;
HMAC_SHA1(pw, pw_len, s, salt_len + 4, u);
memcpy(t, u, sizeof(t));
for (j = 1; j < iterations; j++)
{
HMAC_SHA1(pw, pw_len, u, sizeof(u), u);
for (n = 0; n < h_len; n++)
t[n] ^= u[n];
}
for (n = 0; n < h_len && (n + k) < dk_len; n++)
derived_key[n + k] = t[n];
}
}
void PBKDF2_HMAC_SHA256(const uint8_t* pw, size_t pw_len, const uint8_t* salt, size_t salt_len, int iterations, uint8_t* derived_key, size_t dk_len)
{
// HMAC_SHA256(pw, key_len, salt, salt_len, mac)
// l = ceil(dkLen / hLen); // No of blocks
// r = dkLen - (l - 1) * hLen; // Octets in last block
// T[k] = F(P, S, c, k + 1); // k = 1 .. iterations
// F(P, S, c, k) = U_1 ^ U_2 ^ ... U_c
// U_1 = PRF(P, S || INT(i))
// U_2 = PRF(P, U_1)
// U_c = PRF(P, U_{c-1})
assert(salt_len <= 0x10);
assert(dk_len <= 0x20);
constexpr size_t h_len = 0x20;
size_t r;
size_t l;
uint8_t t[h_len];
uint8_t u[h_len];
uint8_t s[0x14];
size_t k;
int j;
uint32_t i;
size_t n;
r = dk_len % h_len;
l = dk_len / h_len;
if (r > 0) l++;
for (i = 1, k = 0; k < dk_len; i++, k += h_len)
{
// F(P,S,c,i)
memcpy(s, salt, salt_len);
s[salt_len + 0] = (i >> 24) & 0xFF;
s[salt_len + 1] = (i >> 16) & 0xFF;
s[salt_len + 2] = (i >> 8) & 0xFF;
s[salt_len + 3] = i & 0xFF;
HMAC_SHA256(pw, pw_len, s, salt_len + 4, u);
memcpy(t, u, sizeof(t));
for (j = 1; j < iterations; j++)
{
HMAC_SHA256(pw, pw_len, u, sizeof(u), u);
for (n = 0; n < h_len; n++)
t[n] ^= u[n];
}
for (n = 0; n < h_len && (n + k) < dk_len; n++)
derived_key[n + k] = t[n];
}
}
+12
查看文件
@@ -0,0 +1,12 @@
#pragma once
#include <cstdint>
#include "Aes.h"
void Rfc3394_KeyWrap(uint8_t *crypto, const uint8_t *plain, size_t size, const uint8_t *key, AES::Mode aes_mode, uint64_t iv);
bool Rfc3394_KeyUnwrap(uint8_t *plain, const uint8_t *crypto, size_t size, const uint8_t *key, AES::Mode aes_mode, uint64_t *iv);
void HMAC_SHA1(const uint8_t *key, size_t key_len, const uint8_t *data, size_t data_len, uint8_t *mac);
void HMAC_SHA256(const uint8_t *key, size_t key_len, const uint8_t *data, size_t data_len, uint8_t *mac);
void PBKDF2_HMAC_SHA1(const uint8_t* pw, size_t pw_len, const uint8_t* salt, size_t salt_len, int iterations, uint8_t* derived_key, size_t dk_len);
void PBKDF2_HMAC_SHA256(const uint8_t* pw, size_t pw_len, const uint8_t* salt, size_t salt_len, int iterations, uint8_t* derived_key, size_t dk_len);
已加载 100/111 个文件。由于本次变更涉及的文件过多,其余文件未显示。 显示更多