diff --git a/aperisolve/analyzers/utils.py b/aperisolve/analyzers/utils.py index a20036a..693ea94 100644 --- a/aperisolve/analyzers/utils.py +++ b/aperisolve/analyzers/utils.py @@ -6,6 +6,9 @@ import os import threading from pathlib import Path from typing import Any +import itertools +import zlib +import sqlite3 _thread_lock = threading.Lock() @@ -46,3 +49,85 @@ def update_data( os.replace(tmp_file, json_file) # ensures file write is atomic finally: fcntl.flock(lock, fcntl.LOCK_UN) + + +def create_crc_db(): + db_filename = "ihdr_crcs.db" + + # Connect to database + conn = sqlite3.connect(db_filename) + cursor = conn.cursor() + + # Optimization: Turn off safety checks for bulk insertion speed + cursor.execute("PRAGMA synchronous = OFF") + cursor.execute("PRAGMA journal_mode = MEMORY") + + # Create table + # We store the CRC as an integer for faster indexing/storage + cursor.execute(""" + CREATE TABLE IF NOT EXISTS ihdr ( + crc INTEGER, + width INTEGER, + height INTEGER + ) + """) + + # Drop index if exists (rebuilding it at the end is faster) + cursor.execute("DROP INDEX IF EXISTS idx_crc") + + max_width = 2000 + max_height = 2000 + W = range(1, max_width + 1) + H = range(1, max_height + 1) + + # Standard PNG IHDR parameters + bit_depth = [1, 2, 4, 8, 16] + color_type = [0, 2, 3, 4, 6] + compression_method = [0] # This is always 0 according to the PNG spec + filter_method = [0] # This is always 0 according to the PNG spec + interlace_method = [0, 1] + + # Create the generator + all_combinations = itertools.product( + W, H, bit_depth, color_type, compression_method, filter_method, interlace_method + ) + + batch_size = 100000 + batch = [] + count = 0 + + for w, h, bd, ct, comp, filt, inter in all_combinations: + # Construct IHDR chunk data + ihdr_data = ( + w.to_bytes(4, "big") + + h.to_bytes(4, "big") + + bytes([bd, ct, comp, filt, inter]) + ) + + # Calculate CRC + crc = zlib.crc32(b"IHDR" + ihdr_data) & 0xffffffff + + # Add to batch + batch.append((crc, w, h)) + + if len(batch) >= batch_size: + cursor.executemany("INSERT INTO ihdr VALUES (?,?,?)", batch) + conn.commit() + batch = [] + count += batch_size + if count % 1000000 == 0: + print(f"Inserted {count:,} rows...") + + # Insert remaining records + if batch: + cursor.executemany("INSERT INTO ihdr VALUES (?,?,?)", batch) + conn.commit() + + print("Insertion complete. Creating Index for faster lookups...") + # Creating the index allows for faster lookups + cursor.execute("CREATE INDEX idx_crc ON ihdr(crc)") + conn.commit() + + conn.close() + print(f"Database ready.") + return \ No newline at end of file