Merge pull request #106 from Philip-Zimmermann/feature/png_crc_db_lookup
CRC Database lookup for image_resize.py, fix #96
This commit is contained in:
Zeecka
@@ -7,4 +7,4 @@ DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve
|
||||
RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0
|
||||
POSTGRES_DB=aperisolve
|
||||
POSTGRES_USER=aperiuser
|
||||
POSTGRES_PASSWORD=aperipass
|
||||
POSTGRES_PASSWORD=aperipass
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
.mypy_cache/
|
||||
results/
|
||||
results/
|
||||
|
||||
+5
-1
@@ -19,6 +19,10 @@ RUN apt-get update && apt-get install -y \
|
||||
|
||||
COPY aperisolve/ /aperisolve/
|
||||
|
||||
RUN mkdir -p /data
|
||||
|
||||
RUN pip install --no-cache-dir -r /aperisolve/requirements.txt
|
||||
|
||||
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "aperisolve.wsgi:application"]
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.wsgi:application"]
|
||||
@@ -24,7 +24,7 @@ def analyze_foremost(input_img: Path, output_dir: Path) -> None:
|
||||
timeout=MAX_PENDING_TIME,
|
||||
)
|
||||
# Note: foremost use stderr as standard output for results :o)
|
||||
# So we can't use it as error dectection :
|
||||
# So we can't use it as error detection :
|
||||
# stderr += data.stderr
|
||||
|
||||
# Zip extracted files
|
||||
|
||||
@@ -1,165 +1,118 @@
|
||||
"""Image size bruteforcer for common PNG sizes."""
|
||||
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from .utils import update_data
|
||||
from typing import List
|
||||
|
||||
# Full list of expected sizes
|
||||
EXPECTED_SIZES = [
|
||||
# -------------------------
|
||||
# ICONS (Square)
|
||||
# -------------------------
|
||||
(16, 16),
|
||||
(20, 20),
|
||||
(24, 24),
|
||||
(28, 28),
|
||||
(32, 32),
|
||||
(48, 48),
|
||||
(64, 64),
|
||||
(72, 72),
|
||||
(96, 96),
|
||||
(128, 128),
|
||||
(192, 192),
|
||||
(256, 256),
|
||||
(384, 384),
|
||||
(512, 512),
|
||||
(768, 768),
|
||||
(1024, 1024),
|
||||
(1536, 1536),
|
||||
(2048, 2048),
|
||||
(4096, 4096),
|
||||
# -------------------------
|
||||
# 1:1 Square (General)
|
||||
# -------------------------
|
||||
(300, 300),
|
||||
(500, 500),
|
||||
(800, 800),
|
||||
(1000, 1000),
|
||||
(1500, 1500),
|
||||
(2000, 2000),
|
||||
(2500, 2500),
|
||||
(3000, 3000),
|
||||
# -------------------------
|
||||
# 4:3 Aspect (Classic Screens)
|
||||
# -------------------------
|
||||
(320, 240),
|
||||
(640, 480),
|
||||
(800, 600),
|
||||
(960, 720),
|
||||
(1024, 768),
|
||||
(1280, 960),
|
||||
(1600, 1200),
|
||||
(2048, 1536),
|
||||
(2560, 1920),
|
||||
(3200, 2400),
|
||||
(4096, 3072),
|
||||
# -------------------------
|
||||
# 3:2 Aspect (Photography)
|
||||
# -------------------------
|
||||
(600, 400),
|
||||
(900, 600),
|
||||
(1200, 800),
|
||||
(1500, 1000),
|
||||
(1800, 1200),
|
||||
(2400, 1600),
|
||||
(3000, 2000),
|
||||
(3600, 2400),
|
||||
(4500, 3000),
|
||||
(6000, 4000),
|
||||
# -------------------------
|
||||
# 16:9 Aspect (Video / Web)
|
||||
# -------------------------
|
||||
(426, 240),
|
||||
(640, 360),
|
||||
(854, 480),
|
||||
(960, 540),
|
||||
(1280, 720),
|
||||
(1366, 768),
|
||||
(1600, 900),
|
||||
(1920, 1080),
|
||||
(2560, 1440),
|
||||
(3200, 1800),
|
||||
(3440, 1935),
|
||||
(3840, 2160),
|
||||
(5120, 2880),
|
||||
(7680, 4320),
|
||||
(10240, 5760),
|
||||
# -------------------------
|
||||
# 16:10 Aspect (Laptops)
|
||||
# -------------------------
|
||||
(1280, 800),
|
||||
(1440, 900),
|
||||
(1680, 1050),
|
||||
(1920, 1200),
|
||||
(2560, 1600),
|
||||
(2880, 1800),
|
||||
(3840, 2400),
|
||||
(5120, 3200),
|
||||
# -------------------------
|
||||
# Mobile Portrait (9:16)
|
||||
# -------------------------
|
||||
(720, 1280),
|
||||
(1080, 1920),
|
||||
(1440, 2560),
|
||||
(2160, 3840),
|
||||
(1440, 2960),
|
||||
(1080, 2400),
|
||||
(1170, 2532),
|
||||
(1284, 2778),
|
||||
(1290, 2796),
|
||||
# -------------------------
|
||||
# Social Media Sizes
|
||||
# -------------------------
|
||||
# Instagram
|
||||
(1080, 1080),
|
||||
(1080, 1350),
|
||||
(1080, 1920),
|
||||
# Facebook
|
||||
(1200, 628),
|
||||
(1200, 1200),
|
||||
# Twitter/X
|
||||
(1200, 675),
|
||||
(1500, 500),
|
||||
# YouTube Thumbnails
|
||||
(1280, 720),
|
||||
(1920, 1080),
|
||||
# -------------------------
|
||||
# Banners / Headers
|
||||
# -------------------------
|
||||
(1600, 500),
|
||||
(1920, 500),
|
||||
(2560, 700),
|
||||
(3000, 1000),
|
||||
(3840, 1200),
|
||||
# -------------------------
|
||||
# Print-like Digital Sizes
|
||||
# -------------------------
|
||||
(2550, 3300), # Letter @ 300 DPI
|
||||
(2480, 3508), # A4 @ 300 DPI
|
||||
(3508, 4961), # A3 @ 300 DPI
|
||||
(4961, 7016), # A2 @ 300 DPI
|
||||
(7016, 9933), # A1 @ 300 DPI
|
||||
]
|
||||
from ..app import app
|
||||
from ..models import IHDR
|
||||
from .utils import PNG, update_data
|
||||
|
||||
|
||||
def calc_checksum(
|
||||
header_chunk: bytearray, width_bytes: bytearray, height_bytes: bytearray
|
||||
) -> bytearray:
|
||||
def write_recovered_png(
|
||||
png_bytes: bytearray,
|
||||
width: int,
|
||||
height: int,
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Calculates the CRC32 of the IHDR chunk with new dimensions.
|
||||
header_chunk: The full IHDR chunk (Type + Data).
|
||||
Rebuilds a PNG by patching IHDR width/height and writes it to disk.
|
||||
"""
|
||||
# Reconstruct IHDR: [Type(4)] + [New Width(4)] + [New Height(4)] + [Rest of Data(5)]
|
||||
new_header = header_chunk[:4] + width_bytes + height_bytes + header_chunk[12:]
|
||||
height_bytes = height.to_bytes(4, byteorder="big")
|
||||
width_bytes = width.to_bytes(4, byteorder="big")
|
||||
|
||||
# Calculate CRC and return as 4 bytes (Big Endian)
|
||||
return bytearray((zlib.crc32(new_header) & 0xFFFFFFFF).to_bytes(4, byteorder="big"))
|
||||
full_png_data = (
|
||||
png_bytes[:16] # Splicing: [Start...IHDR+4] + [W] + [H] + [IHDR+12...End]
|
||||
+ width_bytes
|
||||
+ height_bytes
|
||||
+ png_bytes[24:]
|
||||
)
|
||||
|
||||
with output_path.open("wb") as out_f:
|
||||
out_f.write(full_png_data)
|
||||
|
||||
|
||||
def lookup_crc(png: PNG) -> List[PNG]:
|
||||
"""
|
||||
Look up PNG images in the database by their CRC value.
|
||||
|
||||
Args:
|
||||
png (PNG): A PNG object containing the CRC value to search for.
|
||||
|
||||
Returns:
|
||||
List[PNG]: A list of PNG objects with matching CRC values, reconstructed from their packed
|
||||
representations.
|
||||
"""
|
||||
with app.app_context():
|
||||
imgs = IHDR.query.filter_by(crc=png.crc).all()
|
||||
results = []
|
||||
for img in imgs:
|
||||
packed = img.packed
|
||||
results.append(PNG.from_packed(packed, crc=png.crc))
|
||||
return results
|
||||
|
||||
|
||||
def search_height_crc(png: PNG) -> List[PNG]:
|
||||
"""
|
||||
Search for PNG images with matching CRC by iterating through possible heights.
|
||||
|
||||
This function attempts to find alternative PNG configurations that produce the same
|
||||
CRC checksum as the input PNG by systematically varying the height parameter while
|
||||
keeping all other properties constant.
|
||||
|
||||
Args:
|
||||
png (PNG): The reference PNG object whose CRC checksum will be matched.
|
||||
|
||||
Returns:
|
||||
List[PNG]: A list of PNG objects with matching CRC values. Returns an empty list
|
||||
if no matches are found.
|
||||
|
||||
Note:
|
||||
The function iterates through heights from 100 to 3500 (inclusive).
|
||||
This may be computationally expensive for large ranges.
|
||||
"""
|
||||
results = []
|
||||
for height in range(100, 3501):
|
||||
candidate = PNG(
|
||||
width=png.width,
|
||||
height=height,
|
||||
bit_depth=png.bit_depth,
|
||||
color_type=png.color_type,
|
||||
interlace=png.interlace,
|
||||
)
|
||||
if candidate.crc == png.crc:
|
||||
results.append(candidate)
|
||||
return results
|
||||
|
||||
|
||||
def analyze_image_resize(input_img: Path, output_dir: Path) -> None:
|
||||
"""
|
||||
Analyze an image submission using python-based resize bruteforce.
|
||||
Attempts to recover PNGs with modified resolutions but valid CRCs.
|
||||
Analyze a PNG image and recover resized versions based on CRC matching.
|
||||
|
||||
This function attempts to find matching image dimensions for a given PNG file
|
||||
by looking up its CRC (Cyclic Redundancy Check) in a database. If matches are
|
||||
found, it recovers and saves PNG images with the matched dimensions.
|
||||
|
||||
Args:
|
||||
input_img (Path): Path to the input PNG image file to analyze.
|
||||
output_dir (Path): Path to the directory where recovered PNG images will be saved.
|
||||
|
||||
Returns:
|
||||
None
|
||||
|
||||
Raises:
|
||||
Handles exceptions gracefully by updating output data with error status.
|
||||
|
||||
Side Effects:
|
||||
- Creates output directory if it doesn't exist.
|
||||
- Saves recovered PNG images to the output directory.
|
||||
- Updates data in the output directory with analysis results via update_data().
|
||||
|
||||
Notes:
|
||||
- If the PNG has invalid structure or IHDR chunk is not first, an error is logged.
|
||||
- The function performs a two-step lookup: first by CRC, then by height/CRC if no matches
|
||||
found.
|
||||
- Recovered images are named as "recovered_{width}x{height}.png".
|
||||
- Analysis results (logs, status, and image URLs) are persisted via update_data().
|
||||
"""
|
||||
|
||||
# 1. Setup Paths
|
||||
@@ -168,81 +121,61 @@ def analyze_image_resize(input_img: Path, output_dir: Path) -> None:
|
||||
|
||||
# 3. Initialize Logs
|
||||
logs = []
|
||||
recovered_image = None
|
||||
|
||||
try:
|
||||
with input_img.open("rb") as image:
|
||||
f = image.read()
|
||||
img_bytes = bytearray(image.read())
|
||||
png = PNG.from_bytearray(img_bytes)
|
||||
|
||||
b = bytearray(f)
|
||||
|
||||
# Find IHDR start
|
||||
ihdr = b.find(b"\x49\x48\x44\x52")
|
||||
if ihdr == -1:
|
||||
logs.append("Failure: PNG header (IHDR) not found.")
|
||||
if png is None:
|
||||
logs.append(
|
||||
"Failure: IHDR chunk is not the first chunk, or PNG has invalid structure."
|
||||
)
|
||||
update_data(
|
||||
output_dir,
|
||||
{"image_resize": {"status": "error", "error": "PNG header not found."}},
|
||||
{
|
||||
"image_resize": {
|
||||
"status": "error",
|
||||
"error": "IHDR chunk is not the first chunk, or PNG has invalid structure.",
|
||||
}
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
# Calculate Chunk Length (The 4 bytes BEFORE IHDR tag)
|
||||
chunk_length = int.from_bytes((b[ihdr - 4 : ihdr]), byteorder="big") + 4
|
||||
logs.append(f"Target CRC found: 0x{png.crc:08x}")
|
||||
|
||||
# Extract Target CRC (The 4 bytes AFTER the chunk data)
|
||||
target_crc = b[ihdr + chunk_length : ihdr + chunk_length + 4]
|
||||
saved_img_urls = []
|
||||
|
||||
# Isolate the header chunk (Type + Data)
|
||||
header_chunk = b[ihdr : ihdr + chunk_length]
|
||||
db_matches = lookup_crc(png)
|
||||
if not db_matches:
|
||||
db_matches = search_height_crc(png)
|
||||
if not db_matches:
|
||||
logs.append("Failure: No matching dimensions found.")
|
||||
else:
|
||||
# Create parent directories if they don't exist
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logs.append(f"Target CRC found: 0x{target_crc.hex()}")
|
||||
|
||||
match_found = False
|
||||
|
||||
# Brute-force Loop
|
||||
for size in EXPECTED_SIZES:
|
||||
width, height = size
|
||||
|
||||
# Convert dimensions to 4-byte Big Endian arrays
|
||||
width_bytes = bytearray(width.to_bytes(4, byteorder="big"))
|
||||
height_bytes = bytearray(height.to_bytes(4, byteorder="big"))
|
||||
|
||||
# Check if this size matches the file's CRC
|
||||
if target_crc == calc_checksum(header_chunk, width_bytes, height_bytes):
|
||||
match_found = True
|
||||
logs.append(f"Success: Match found! Dimensions: {width}x{height}")
|
||||
|
||||
# Create Output Filename
|
||||
img_name = f"recovered_{width}x{height}.png"
|
||||
|
||||
# Create parent directories if they don't exist
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
# Iterate through ALL candidates found
|
||||
for im_match in db_matches:
|
||||
img_name = f"recovered_{im_match.width}x{im_match.height}.png"
|
||||
output_path = output_dir / img_name
|
||||
|
||||
# Splicing: [Start...IHDR+4] + [W] + [H] + [IHDR+12...End]
|
||||
full_png_data = (
|
||||
b[: ihdr + 4] + width_bytes + height_bytes + b[ihdr + 12 :]
|
||||
write_recovered_png(
|
||||
img_bytes, im_match.width, im_match.height, output_path
|
||||
)
|
||||
|
||||
# Write the recovered file to disk
|
||||
with output_path.open("wb") as out_f:
|
||||
out_f.write(full_png_data)
|
||||
|
||||
recovered_image = Path(output_dir.name) / img_name
|
||||
break # Stop looking after finding the match
|
||||
|
||||
if not match_found:
|
||||
logs.append("Failure: No matching dimensions found.")
|
||||
logs.append(f"Image saved: {img_name}")
|
||||
# Append the formatted URL to our list
|
||||
saved_img_urls.append("/image/" + str(Path(output_dir.name) / img_name))
|
||||
|
||||
# 4. Final Data Update
|
||||
output_data = {
|
||||
"image_resize": { # Key name must match your module name in AperiSolve
|
||||
"image_resize": {
|
||||
"status": "ok",
|
||||
"output": logs,
|
||||
"png_images": saved_img_urls,
|
||||
}
|
||||
}
|
||||
if recovered_image:
|
||||
output_data["image_resize"]["image"] = "/image/" + str(recovered_image)
|
||||
|
||||
update_data(output_dir, output_data)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -11,7 +11,7 @@ from .utils import MAX_PENDING_TIME, update_data
|
||||
def analyze_outguess(
|
||||
input_img: Path, output_dir: Path, password: Optional[str] = None
|
||||
) -> None:
|
||||
"""Analyze an image submission using stehide."""
|
||||
"""Analyze an image submission using outguess."""
|
||||
|
||||
image_name = input_img.name
|
||||
try:
|
||||
|
||||
@@ -7,7 +7,7 @@ from .utils import MAX_PENDING_TIME, update_data
|
||||
|
||||
|
||||
def analyze_pngcheck(input_img: Path, output_dir: Path) -> None:
|
||||
"""Analyze an image submission using strings."""
|
||||
"""Analyze an image submission using pngcheck."""
|
||||
|
||||
try:
|
||||
data = subprocess.run(
|
||||
@@ -24,15 +24,15 @@ def analyze_pngcheck(input_img: Path, output_dir: Path) -> None:
|
||||
# )
|
||||
# return
|
||||
|
||||
data_strings = data.stdout.split("\n") if data else []
|
||||
data_strings = [s for s in data_strings if s]
|
||||
data_pngcheck = data.stdout.split("\n") if data else []
|
||||
data_pngcheck = [s for s in data_pngcheck if s]
|
||||
|
||||
update_data(
|
||||
output_dir,
|
||||
{
|
||||
"pngcheck": {
|
||||
"status": "ok",
|
||||
"output": data_strings,
|
||||
"output": data_pngcheck,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ from .utils import MAX_PENDING_TIME, update_data
|
||||
def analyze_steghide(
|
||||
input_img: Path, output_dir: Path, password: Optional[str] = None
|
||||
) -> None:
|
||||
"""Analyze an image submission using stehide."""
|
||||
"""Analyze an image submission using steghide."""
|
||||
|
||||
image_name = input_img.name
|
||||
try:
|
||||
|
||||
@@ -2,10 +2,14 @@
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import threading
|
||||
import zlib
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Self
|
||||
|
||||
_thread_lock = threading.Lock()
|
||||
|
||||
@@ -46,3 +50,255 @@ def update_data(
|
||||
os.replace(tmp_file, json_file) # ensures file write is atomic
|
||||
finally:
|
||||
fcntl.flock(lock, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def get_resolutions():
|
||||
"""
|
||||
Generate a sorted list of common display and image resolutions.
|
||||
|
||||
Creates a comprehensive set of resolution tuples by combining various base widths
|
||||
with standard aspect ratios used in screens, digital displays, photography, and
|
||||
paper standards. Includes both portrait and landscape orientations.
|
||||
|
||||
Base widths range from 16px to 10000px, covering:
|
||||
- Low resolution: 16-256px (16px increments)
|
||||
- Standard: 320-1024px (32px increments)
|
||||
- HD/Full HD: 1280-2560px (64px increments)
|
||||
- 4K: 3000-4096px (128px increments)
|
||||
- 5K-8K: 5120-8192px (256px increments)
|
||||
- Upper bound: 10000px
|
||||
|
||||
Aspect ratios include common formats:
|
||||
- Digital screens: 1:1, 4:3, 3:2, 16:10, 16:9, 21:9
|
||||
- Photography/print: 5:4, 7:5, 2:3
|
||||
- Paper standards: ISO A-series, US Letter, Legal, Tabloid
|
||||
|
||||
Returns:
|
||||
list[tuple[int, int]]: Sorted list of (width, height) resolution tuples
|
||||
with heights constrained between 1 and 10000 pixels.
|
||||
"""
|
||||
base_widths = (
|
||||
list(range(16, 257, 16)) # 16 → 256
|
||||
+ list(range(320, 1025, 32)) # 320 → 1024
|
||||
+ list(range(1280, 2561, 64)) # 1280 → 2560
|
||||
+ list(range(3000, 4097, 128)) # 3K–4K
|
||||
+ list(range(5120, 8193, 256)) # 5K–8K
|
||||
+ [10000] # upper bound
|
||||
)
|
||||
aspect_ratios = [
|
||||
# Screens / digital
|
||||
(1, 1), # square
|
||||
(4, 3),
|
||||
(3, 2),
|
||||
(16, 10),
|
||||
(16, 9),
|
||||
(21, 9),
|
||||
# Photography / print
|
||||
(5, 4), # 8x10
|
||||
(7, 5), # 5x7
|
||||
(3, 2), # 4x6 (already included, kept for clarity)
|
||||
(2, 3), # poster
|
||||
# Paper standards
|
||||
(1414, 1000), # ISO A-series (A4, A3, A2, ...)
|
||||
(11, 85), # US Letter
|
||||
(14, 85), # Legal
|
||||
(17, 11), # Tabloid
|
||||
]
|
||||
resolutions = set()
|
||||
for w in base_widths:
|
||||
for ar_w, ar_h in aspect_ratios:
|
||||
h = math.floor(round(w * ar_h / ar_w))
|
||||
if 1 <= h <= 10000:
|
||||
resolutions.add((w, h))
|
||||
resolutions.add((h, w)) # portrait / landscape
|
||||
return sorted(resolutions)
|
||||
|
||||
|
||||
def get_valid_depth_color_pairs():
|
||||
"""
|
||||
Generate valid combinations of color and bit depth values.
|
||||
|
||||
Yields tuples of (depth, color) representing valid color mode and bit depth
|
||||
combinations. Supports grayscale (0), RGB (2, 3, 4, 6) color modes with
|
||||
appropriate bit depths (1, 2, 4, 8, 16 bits).
|
||||
|
||||
Yields:
|
||||
tuple: A tuple of (depth, color) where depth is an integer representing
|
||||
bit depth and color is an integer representing the color mode.
|
||||
"""
|
||||
valid_color_depth = {
|
||||
0: [1, 2, 4, 8, 16],
|
||||
2: [8, 16],
|
||||
3: [1, 2, 4, 8],
|
||||
4: [8, 16],
|
||||
6: [8, 16],
|
||||
}
|
||||
for color, depths in valid_color_depth.items():
|
||||
for depth in depths:
|
||||
yield depth, color
|
||||
|
||||
|
||||
class PNG:
|
||||
"""
|
||||
A class representing a PNG image header (IHDR chunk).
|
||||
This class encapsulates the metadata from a PNG file's IHDR (Image Header) chunk,
|
||||
including image dimensions, bit depth, color type, interlacing method, and CRC checksum.
|
||||
It provides methods for parsing PNG files, packing/unpacking header data, and generating
|
||||
PNG crc if omitted.
|
||||
"""
|
||||
|
||||
width: int
|
||||
height: int
|
||||
bit_depth: int
|
||||
color_type: int
|
||||
crc: int
|
||||
interlace: int = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
width: int,
|
||||
height: int,
|
||||
bit_depth: int,
|
||||
color_type: int,
|
||||
crc: int | None = None,
|
||||
interlace: int = 0,
|
||||
):
|
||||
"""Initialize a PNG IHDR (Image Header) chunk.
|
||||
|
||||
Args:
|
||||
width (int): The width of the image in pixels.
|
||||
height (int): The height of the image in pixels.
|
||||
bit_depth (int): The number of bits per sample or per palette index (1, 2, 4, 8, or 16).
|
||||
color_type (int): The color type of the image (0=Grayscale, 2=RGB, 3=Indexed,
|
||||
4=Grayscale+Alpha, 6=RGBA).
|
||||
crc (int | None, optional): CRC (Cyclic Redundancy Check) checksum for the chunk.
|
||||
If None, it will be calculated automatically. Defaults
|
||||
to None.
|
||||
interlace (int, optional): The interlace method (0=no interlace, 1=Adam7 interlace).
|
||||
Defaults to 0.
|
||||
"""
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.bit_depth = bit_depth
|
||||
self.color_type = color_type
|
||||
self.crc = (
|
||||
crc
|
||||
or zlib.crc32(
|
||||
b"IHDR"
|
||||
+ width.to_bytes(4, "big")
|
||||
+ height.to_bytes(4, "big")
|
||||
+ bytes((bit_depth, color_type, 0, 0, interlace))
|
||||
)
|
||||
& 0xFFFFFFFF
|
||||
)
|
||||
self.interlace = interlace
|
||||
|
||||
@classmethod
|
||||
def from_packed(cls, packed: int, crc: int | None = None) -> Self:
|
||||
"""Create a PNG instance from a packed integer representation.
|
||||
|
||||
Parse a packed integer containing width, height, bit depth, color type and interlace
|
||||
bits and return a PNG instance. The CRC may be supplied; if omitted it will be
|
||||
computed by the constructor.
|
||||
|
||||
Args:
|
||||
packed (int): Packed integer with fields laid out as described in the `packed` property.
|
||||
crc (int | None): Optional CRC value for the IHDR chunk.
|
||||
|
||||
Returns:
|
||||
Self: PNG instance with parsed fields.
|
||||
"""
|
||||
interlace = packed & 0b1
|
||||
color = (packed >> 1) & 0b111
|
||||
depth = (packed >> 4) & 0b111
|
||||
height = (packed >> 7) & 0xFFFF
|
||||
width = (packed >> 23) & 0xFFFF
|
||||
return cls(
|
||||
width=width,
|
||||
height=height,
|
||||
bit_depth=depth,
|
||||
color_type=color,
|
||||
crc=crc,
|
||||
interlace=interlace,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def packed(self) -> int:
|
||||
"""Pack IHDR fields into a single integer.
|
||||
|
||||
Returns:
|
||||
int: Packed integer encoding the IHDR fields.
|
||||
|
||||
Layout:
|
||||
width : 16 bits
|
||||
height : 16 bits
|
||||
depth : 3 bits
|
||||
color : 3 bits
|
||||
interlace : 1 bit
|
||||
"""
|
||||
return (
|
||||
((self.width & 0xFFFF) << 23)
|
||||
| ((self.height & 0xFFFF) << 7)
|
||||
| ((self.bit_depth & 0b111) << 4)
|
||||
| ((self.color_type & 0b111) << 1)
|
||||
| (self.interlace & 0b1)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_bytearray(cls, data: bytearray) -> Self:
|
||||
"""Create a PNG image instance from a bytearray containing PNG file data.
|
||||
|
||||
Parses the PNG signature and IHDR (Image Header) chunk to extract image metadata
|
||||
including dimensions, bit depth, color type, and interlacing information.
|
||||
|
||||
Args:
|
||||
data (bytearray): A bytearray containing the complete PNG file data, starting
|
||||
with the PNG signature (8 bytes: \\x89PNG\\r\\n\\x1a\\n).
|
||||
|
||||
Returns:
|
||||
Self | None: A new instance of the PNG image class with parsed IHDR metadata
|
||||
(width, height, bit_depth, color_type, crc, interlace) if the data is
|
||||
valid and contains a proper IHDR chunk. Returns None if the PNG signature
|
||||
is invalid or the IHDR chunk type is not found.
|
||||
|
||||
Raises:
|
||||
struct.error: If the data is too short or malformed for unpacking IHDR fields.
|
||||
IndexError: If the bytearray is shorter than expected for the PNG structure.
|
||||
"""
|
||||
|
||||
png_signature = b"\x89PNG\r\n\x1a\n"
|
||||
if data[:8] != png_signature:
|
||||
raise ValueError("Invalid PNG signature")
|
||||
offset = 8 # Read IHDR chunk
|
||||
length = struct.unpack(">I", data[offset : offset + 4])[0]
|
||||
offset += 4
|
||||
chunk_type = data[offset : offset + 4]
|
||||
offset += 4
|
||||
if chunk_type != b"IHDR":
|
||||
raise ValueError("IHDR chunk not found at expected position")
|
||||
|
||||
ihdr_data = data[offset : offset + length]
|
||||
offset += length
|
||||
|
||||
crc = struct.unpack(">I", data[offset : offset + 4])[0]
|
||||
|
||||
# Unpack IHDR fields
|
||||
width, height, depth, color, _, _, interlace = struct.unpack(
|
||||
">IIBBBBB", ihdr_data
|
||||
)
|
||||
|
||||
return cls(
|
||||
width=width,
|
||||
height=height,
|
||||
bit_depth=depth,
|
||||
color_type=color,
|
||||
crc=crc,
|
||||
interlace=interlace,
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"PNG(width={self.width}, height={self.height}, "
|
||||
f"bit_depth={self.bit_depth}, color_type={self.color_type}, "
|
||||
f"interlace={self.interlace}, crc=0x{self.crc:08x})"
|
||||
)
|
||||
|
||||
+7
-7
@@ -1,4 +1,4 @@
|
||||
"""AperiSolve Flask application."""
|
||||
"""Aperi'Solve Flask application."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
@@ -41,7 +41,7 @@ def too_large(_: Any) -> tuple[Response, int]:
|
||||
@app.errorhandler(404)
|
||||
def not_found(_: Any) -> str:
|
||||
"""Error Handler for 404 not found."""
|
||||
return render_template("error.html", message="Ressource not found", statuscode=404)
|
||||
return render_template("error.html", message="Resource not found", statuscode=404)
|
||||
|
||||
|
||||
@app.route("/")
|
||||
@@ -134,12 +134,12 @@ def upload_image() -> tuple[Response, int]:
|
||||
first_submission_date=datetime.now(timezone.utc),
|
||||
last_submission_date=datetime.now(timezone.utc),
|
||||
)
|
||||
db.session.add(new_img)
|
||||
db.session.commit()
|
||||
db.session.add(new_img) # pylint: disable=no-member
|
||||
db.session.commit() # pylint: disable=no-member
|
||||
|
||||
sub_img = Image.query.filter_by(hash=img_hash).first() # type: ignore
|
||||
sub_img.upload_count += 1
|
||||
db.session.commit()
|
||||
db.session.commit() # pylint: disable=no-member
|
||||
|
||||
# Create new Submission entry
|
||||
submission_path.mkdir(parents=True, exist_ok=True)
|
||||
@@ -152,9 +152,9 @@ def upload_image() -> tuple[Response, int]:
|
||||
date=time.time(),
|
||||
image_hash=sub_img.hash, # type: ignore
|
||||
)
|
||||
db.session.add(submission)
|
||||
db.session.add(submission) # pylint: disable=no-member
|
||||
|
||||
db.session.commit() # Commit to save the new Image and Submission
|
||||
db.session.commit() # pylint: disable=no-member
|
||||
# Start the analysis jobs
|
||||
queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300)
|
||||
|
||||
|
||||
+28
-7
@@ -14,7 +14,28 @@ MAX_STORE_TIME = int(getenv("MAX_STORE_TIME", "259200")) # 3 days by default
|
||||
|
||||
|
||||
def cleanup_old_entries() -> None:
|
||||
"""Clean up old entries in the database and file system."""
|
||||
"""
|
||||
Clean up old and incomplete entries from the database and file system.
|
||||
|
||||
This function performs two main cleanup operations:
|
||||
|
||||
1. Submission cleanup:
|
||||
- Deletes submissions with "pending" or "running" status that have exceeded
|
||||
the maximum allowed processing time (MAX_PENDING_TIME).
|
||||
- Deletes completed submissions ("done" status) that have missing or buggy
|
||||
results (results.json file not found).
|
||||
|
||||
2. Image cleanup:
|
||||
- Deletes images older than MAX_STORE_TIME along with all associated
|
||||
submissions and their result folders.
|
||||
- Deletes orphaned images (with no submissions) older than MAX_PENDING_TIME
|
||||
and removes their associated result folders from the file system.
|
||||
|
||||
All database changes are committed at the end of the operation.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
now = time.time()
|
||||
for submission in Submission.query.all(): # type: ignore
|
||||
if (
|
||||
@@ -22,7 +43,7 @@ def cleanup_old_entries() -> None:
|
||||
and now - submission.date > MAX_PENDING_TIME # type: ignore
|
||||
):
|
||||
# Processing took too long, delete
|
||||
db.session.delete(submission) # type: ignore
|
||||
db.session.delete(submission) # pylint: disable=no-member
|
||||
elif submission.status == "done": # type: ignore
|
||||
# Search for buggy results, delete
|
||||
result_path = (
|
||||
@@ -30,7 +51,7 @@ def cleanup_old_entries() -> None:
|
||||
)
|
||||
result_file = result_path / "results.json"
|
||||
if not result_file.exists():
|
||||
db.session.delete(submission) # type: ignore
|
||||
db.session.delete(submission) # pylint: disable=no-member
|
||||
shutil.rmtree(result_path)
|
||||
|
||||
# Delete "old" images
|
||||
@@ -43,15 +64,15 @@ def cleanup_old_entries() -> None:
|
||||
img_fold = RESULT_FOLDER / img.hash
|
||||
if delay.total_seconds() > MAX_STORE_TIME:
|
||||
for s in img.submissions:
|
||||
db.session.delete(s) # type: ignore
|
||||
db.session.delete(s) # pylint: disable=no-member
|
||||
if img_fold.exists():
|
||||
shutil.rmtree(img_fold) # type: ignore
|
||||
db.session.delete(img)
|
||||
db.session.delete(img) # pylint: disable=no-member
|
||||
|
||||
# Delete Images with missing submission
|
||||
if len(img.submissions) == 0 and delay.total_seconds() > MAX_PENDING_TIME:
|
||||
if img_fold.exists():
|
||||
shutil.rmtree(img_fold) # type: ignore
|
||||
db.session.delete(img)
|
||||
db.session.delete(img) # pylint: disable=no-member
|
||||
|
||||
db.session.commit()
|
||||
db.session.commit() # pylint: disable=no-member
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""AperiSolve configuration variables."""
|
||||
"""Aperi'Solve configuration variables."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
+93
-17
@@ -1,11 +1,13 @@
|
||||
"""This module defines the database models for the Aperisolve application."""
|
||||
"""This module defines the database models for the Aperi'Solve application."""
|
||||
|
||||
import itertools
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
|
||||
from flask import Flask
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from sqlalchemy import Boolean, Column, DateTime, Float, Integer, String
|
||||
from sqlalchemy import BigInteger, Boolean, Column, DateTime, Float, Integer, String
|
||||
|
||||
from aperisolve.analyzers.utils import PNG, get_resolutions, get_valid_depth_color_pairs
|
||||
|
||||
db: SQLAlchemy = SQLAlchemy()
|
||||
|
||||
@@ -13,21 +15,21 @@ db: SQLAlchemy = SQLAlchemy()
|
||||
class Image(db.Model): # type: ignore
|
||||
"""Model representing an image file in the database."""
|
||||
|
||||
hash: Column[str] = Column(
|
||||
hash: Column[String] = Column(
|
||||
String(64), primary_key=True, unique=True, nullable=False
|
||||
)
|
||||
file: Column[str] = Column(String(128), unique=True, nullable=False)
|
||||
size: Column[int] = Column(Integer, nullable=False)
|
||||
first_submission_date: Column[datetime] = Column(
|
||||
file: Column[String] = Column(String(128), unique=True, nullable=False)
|
||||
size: Column[Integer] = Column(Integer, nullable=False)
|
||||
first_submission_date: Column[DateTime] = Column(
|
||||
DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
last_submission_date: Column[datetime] = Column(
|
||||
last_submission_date: Column[DateTime] = Column(
|
||||
DateTime,
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
upload_count: Column[int] = Column(Integer)
|
||||
upload_count: Column[Integer] = Column(Integer)
|
||||
submissions = db.relationship("Submission", backref="image", lazy=True)
|
||||
|
||||
|
||||
@@ -37,14 +39,14 @@ class Submission(db.Model): # type: ignore
|
||||
option.
|
||||
"""
|
||||
|
||||
hash: Column[str] = Column(
|
||||
hash: Column[String] = Column(
|
||||
String(128), primary_key=True, unique=True, nullable=False
|
||||
)
|
||||
filename: Column[str] = Column(String(128), nullable=False)
|
||||
password: Column[str] = Column(String(128))
|
||||
deep_analysis: Column[bool] = Column(Boolean, default=False)
|
||||
status: Column[str] = Column(String(20), default="pending")
|
||||
date: Column[Decimal] = Column(
|
||||
filename: Column[String] = Column(String(128), nullable=False)
|
||||
password: Column[String] = Column(String(128))
|
||||
deep_analysis: Column[Boolean] = Column(Boolean, default=False)
|
||||
status: Column[String] = Column(String(20), default="pending")
|
||||
date: Column[Float] = Column(
|
||||
Float, nullable=False, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
@@ -52,10 +54,84 @@ class Submission(db.Model): # type: ignore
|
||||
image_hash = Column(String, db.ForeignKey("image.hash"), nullable=False)
|
||||
|
||||
|
||||
class IHDR(db.Model): # type: ignore
|
||||
"""IHDR CRC lookup table"""
|
||||
|
||||
iid: Column[Integer] = Column(Integer, primary_key=True, autoincrement=True)
|
||||
crc: Column[BigInteger] = Column(BigInteger, nullable=False)
|
||||
packed: Column[BigInteger] = Column(BigInteger, nullable=False)
|
||||
|
||||
|
||||
def create_crc_db() -> None:
|
||||
"""
|
||||
Creates a database of common PNG IHDR (Image Header) entries by generating
|
||||
combinations of standard PNG parameters such as resolutions, bit depths,
|
||||
color types, and interlace methods. The function checks for existing entries
|
||||
in the database to avoid duplicates and inserts new entries in batches of
|
||||
5000, committing the changes to the database periodically.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
# Standard PNG IHDR parameters
|
||||
resolutions = get_resolutions()
|
||||
bit_color_pairs = get_valid_depth_color_pairs()
|
||||
interlace_methods = [0, 1]
|
||||
|
||||
combinations = itertools.product(
|
||||
resolutions,
|
||||
bit_color_pairs,
|
||||
interlace_methods,
|
||||
)
|
||||
|
||||
count = 0
|
||||
|
||||
with db.session.no_autoflush: # pylint: disable=no-member
|
||||
for (w, h), (bd, ct), inter in combinations:
|
||||
p = PNG(width=w, height=h, bit_depth=bd, color_type=ct, interlace=inter)
|
||||
|
||||
exists = db.session.execute( # pylint: disable=no-member
|
||||
db.select(IHDR.iid).where(IHDR.packed == p.packed)
|
||||
).first()
|
||||
|
||||
if exists: # Skip if already present
|
||||
continue
|
||||
|
||||
db.session.add(IHDR(packed=p.packed, crc=p.crc)) # pylint: disable=no-member
|
||||
count += 1
|
||||
|
||||
if count % 5000 == 0:
|
||||
print(f"Inserted {count} common IHDR entries...")
|
||||
db.session.commit()
|
||||
|
||||
db.session.commit() # pylint: disable=no-member
|
||||
print(f"Precomputed {count} common IHDR entries.")
|
||||
|
||||
|
||||
def init_db(app: Flask) -> None:
|
||||
"""Initialize the database with the given Flask app."""
|
||||
"""
|
||||
Initialize the database by creating tables and populating lookup data.
|
||||
|
||||
This function creates the database schema if it doesn't already exist,
|
||||
and populates the CRC lookup table for PNG IHDR validation.
|
||||
|
||||
Args:
|
||||
app (Flask): The Flask application instance used to establish the application context
|
||||
for database operations.
|
||||
|
||||
Returns:
|
||||
None
|
||||
|
||||
Side Effects:
|
||||
- Creates all database tables if the "image" table doesn't exist
|
||||
- Populates the IHDR CRC lookup table with initial data
|
||||
- Prints status messages to console during initialization
|
||||
"""
|
||||
with app.app_context():
|
||||
if not db.engine.dialect.has_table(db.engine.connect(), "image"):
|
||||
print("Creating database...")
|
||||
db.create_all()
|
||||
print("Database created successfully.")
|
||||
print("Database structure created successfully.")
|
||||
print("Filling IHDR lookup table...")
|
||||
create_crc_db()
|
||||
print("IHDR table filled successfully.")
|
||||
|
||||
@@ -279,6 +279,12 @@ function parseResult(result) {
|
||||
)}'/></div>`;
|
||||
}
|
||||
|
||||
if ("png_images" in result[tool]) {
|
||||
for (const image of result[tool]["png_images"]) {
|
||||
analyzer.innerHTML += `<div class='results_img'><img src='${escapeHtml(image)}'/></div>`;
|
||||
}
|
||||
}
|
||||
|
||||
// Parse text output
|
||||
|
||||
|
||||
|
||||
+29
-5
@@ -8,19 +8,43 @@ from .analyzers.binwalk import analyze_binwalk
|
||||
from .analyzers.decomposer import analyze_decomposer
|
||||
from .analyzers.exiftool import analyze_exiftool
|
||||
from .analyzers.foremost import analyze_foremost
|
||||
from .analyzers.image_resize import analyze_image_resize
|
||||
from .analyzers.outguess import analyze_outguess
|
||||
from .analyzers.pngcheck import analyze_pngcheck
|
||||
from .analyzers.steghide import analyze_steghide
|
||||
from .analyzers.strings import analyze_strings
|
||||
from .analyzers.zsteg import analyze_zsteg
|
||||
from .analyzers.pngcheck import analyze_pngcheck
|
||||
from .analyzers.image_resize import analyze_image_resize
|
||||
from .app import app, db
|
||||
from .config import RESULT_FOLDER
|
||||
from .models import Image, Submission
|
||||
|
||||
|
||||
def analyze_image(submission_hash: str) -> None:
|
||||
"""Analyze an image submission in separate threads."""
|
||||
"""
|
||||
Analyze an image submission by running multiple analysis tools concurrently.
|
||||
|
||||
This function retrieves a submission and its associated image from the database,
|
||||
then executes various image analysis tools in parallel threads. The analysis tools
|
||||
include binwalk, decomposer, exiftool, foremost, strings, pngcheck, steghide,
|
||||
zsteg, and image resize. If deep analysis is enabled, additional tools like
|
||||
outguess are included.
|
||||
|
||||
Args:
|
||||
submission_hash (str): The unique hash identifier of the submission to analyze.
|
||||
|
||||
Returns:
|
||||
None
|
||||
|
||||
Raises:
|
||||
Implicitly catches and handles exceptions during analysis, setting submission
|
||||
status to "error" if an exception occurs.
|
||||
|
||||
Side Effects:
|
||||
- Queries the database for the submission and associated image
|
||||
- Updates submission status in the database ("running" -> "completed" or "error")
|
||||
- Creates result directories and generates analysis output files
|
||||
- Modifies the database session and commits changes
|
||||
"""
|
||||
with app.app_context():
|
||||
submission: Submission = Submission.query.get(submission_hash) # type: ignore
|
||||
image = Image.query.get_or_404(submission.image_hash) # type: ignore
|
||||
@@ -29,7 +53,7 @@ def analyze_image(submission_hash: str) -> None:
|
||||
return
|
||||
|
||||
submission.status = "running" # type: ignore
|
||||
db.session.commit()
|
||||
db.session.commit() # pylint: disable=no-member
|
||||
|
||||
try:
|
||||
img_path: Path = Path(image.file)
|
||||
@@ -77,4 +101,4 @@ def analyze_image(submission_hash: str) -> None:
|
||||
except Exception:
|
||||
submission.status = "error"
|
||||
finally:
|
||||
db.session.commit()
|
||||
db.session.commit() # pylint: disable=no-member
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
Reference in New Issue
Block a user