From 0ecc1812b25b42d36ddf45dea843c636e19a95f0 Mon Sep 17 00:00:00 2001 From: Benjamin Cance Date: Tue, 17 Sep 2024 08:56:35 -0400 Subject: [PATCH] Create the analyzemft/sql directory, add a couple starter tables, modify cli, filewriters, analyzer to (TESTING) output SQLite tables. --- requirements.txt | 3 +- setup.py | 4 ++ src/analyzeMFT/cli.py | 3 ++ src/analyzeMFT/file_writers.py | 59 +++++++++++++++++++++++- src/analyzeMFT/mft_analyzer.py | 43 ++++++++++++++++- src/analyzeMFT/sql/attribute_types.sql | 21 +++++++++ src/analyzeMFT/sql/file_record_flags.sql | 10 ++++ 7 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 src/analyzeMFT/sql/attribute_types.sql create mode 100644 src/analyzeMFT/sql/file_record_flags.sql diff --git a/requirements.txt b/requirements.txt index 7f5213a..1704f18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -openpyxl==3.0.10 \ No newline at end of file +openpyxl==3.0.10 +sqlite3 \ No newline at end of file diff --git a/setup.py b/setup.py index 50a0389..ac14b73 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,9 @@ author_email='bjc@tdx.li', package_dir={'': 'src'}, packages=find_packages(where='src'), + package_data={ + 'analyzeMFT': ['sql/*.sql'], + }, url='http://github.com/rowingdude/analyzeMFT', license='LICENSE.txt', description='Analyze the $MFT from a NTFS filesystem.', @@ -31,6 +34,7 @@ python_requires=">=3.7", install_requires=[ "pywin32;platform_system=='Windows'", + "openpyxl==3.0.10", ], entry_points={ 'console_scripts': [ diff --git a/src/analyzeMFT/cli.py b/src/analyzeMFT/cli.py index fa95ae4..7fa30ac 100644 --- a/src/analyzeMFT/cli.py +++ b/src/analyzeMFT/cli.py @@ -27,6 +27,9 @@ async def main(): help="Export as TSK timeline") export_group.add_option("--l2t", action="store_const", const="l2t", dest="export_format", help="Export as log2timeline CSV") + export_group.add_option("--sqlite", action="store_const", const="sqlite", dest="export_format", + help="Export as SQLite database") + parser.add_option_group(export_group) verbosity_group = OptionGroup(parser, "Verbosity Options") diff --git a/src/analyzeMFT/file_writers.py b/src/analyzeMFT/file_writers.py index 99d177a..d28bf55 100644 --- a/src/analyzeMFT/file_writers.py +++ b/src/analyzeMFT/file_writers.py @@ -1,7 +1,9 @@ +import asyncio import csv +import os import json +import sqlite3 import xml.etree.ElementTree as ET -import asyncio from typing import List, Dict, Any from .mft_record import MftRecord from .constants import * @@ -86,4 +88,57 @@ async def write_l2t(records: List[MftRecord], output_file: str) -> None: date_str, time_str, 'UTC', macb, 'MFT', 'FILESYSTEM', time_type, '', '', '', f"{record.filename} {time_type}", '', record.filename, record.recordnum, '', '', '' ]) - await asyncio.sleep(0) \ No newline at end of file + await asyncio.sleep(0) + + + @staticmethod + async def write_sqlite(records: List[MftRecord], output_file: str) -> None: + conn = sqlite3.connect(output_file) + cursor = conn.cursor() + + # Create and populate static tables + sql_dir = os.path.join(os.path.dirname(__file__), 'sql') + for sql_file in os.listdir(sql_dir): + with open(os.path.join(sql_dir, sql_file), 'r') as f: + cursor.executescript(f.read()) + + # Create MFT records table + cursor.execute(''' + CREATE TABLE mft_records ( + record_number INTEGER PRIMARY KEY, + filename TEXT, + parent_record_number INTEGER, + file_size INTEGER, + is_directory INTEGER, + creation_time TEXT, + modification_time TEXT, + access_time TEXT, + entry_time TEXT, + attribute_types TEXT + ) + ''') + + # Insert MFT records + for record in records: + cursor.execute(''' + INSERT INTO mft_records ( + record_number, filename, parent_record_number, file_size, + is_directory, creation_time, modification_time, access_time, + entry_time, attribute_types + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + record.recordnum, + record.filename, + record.get_parent_record_num(), + record.filesize, + 1 if record.flags & FILE_RECORD_IS_DIRECTORY else 0, + record.fn_times['crtime'].dtstr, + record.fn_times['mtime'].dtstr, + record.fn_times['atime'].dtstr, + record.fn_times['ctime'].dtstr, + ','.join(map(str, record.attribute_types)) + )) + + conn.commit() + conn.close() + await asyncio.sleep(0) \ No newline at end of file diff --git a/src/analyzeMFT/mft_analyzer.py b/src/analyzeMFT/mft_analyzer.py index aee2c8a..43d7283 100644 --- a/src/analyzeMFT/mft_analyzer.py +++ b/src/analyzeMFT/mft_analyzer.py @@ -1,7 +1,9 @@ import asyncio import csv import io +import os import signal +import sqlite3 import sys import traceback from typing import Dict, Set, List, Optional, Any @@ -245,6 +247,8 @@ async def write_output(self) -> None: await FileWriters.write_xml(list(self.mft_records.values()), self.output_file) elif self.export_format == "excel": await FileWriters.write_excel(list(self.mft_records.values()), self.output_file) + elif self.export_format == "sqlite": + await FileWriters.write_sqlite(list(self.mft_records.values()), self.output_file) else: print(f"Unsupported export format: {self.export_format}") @@ -252,4 +256,41 @@ async def cleanup(self): self.log("Performing cleanup...", 1) # to-do add more cleanup after database stuff is integrated. await self.write_remaining_records() - self.log("Cleanup complete.", 1) \ No newline at end of file + self.log("Cleanup complete.", 1) + + async def create_sqlite_database(self): + conn = sqlite3.connect(self.output_file) + cursor = conn.cursor() + + # Create and populate static tables + sql_dir = os.path.join(os.path.dirname(__file__), 'sql') + for sql_file in os.listdir(sql_dir): + with open(os.path.join(sql_dir, sql_file), 'r') as f: + cursor.executescript(f.read()) + + # Create MFT records table + cursor.execute(''' + CREATE TABLE mft_records ( + record_number INTEGER PRIMARY KEY, + filename TEXT, + parent_record_number INTEGER, + -- Add other fields as needed + FOREIGN KEY (attribute_type) REFERENCES attribute_types(id) + ) + ''') + + conn.commit() + return conn + + async def write_sqlite(self): + conn = await self.create_sqlite_database() + cursor = conn.cursor() + + for record in self.mft_records.values(): + cursor.execute(''' + INSERT INTO mft_records (record_number, filename, parent_record_number) + VALUES (?, ?, ?) + ''', (record.recordnum, record.filename, record.get_parent_record_num())) + + conn.commit() + conn.close() \ No newline at end of file diff --git a/src/analyzeMFT/sql/attribute_types.sql b/src/analyzeMFT/sql/attribute_types.sql new file mode 100644 index 0000000..7733bb7 --- /dev/null +++ b/src/analyzeMFT/sql/attribute_types.sql @@ -0,0 +1,21 @@ +CREATE TABLE attribute_types ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL +); + +INSERT INTO attribute_types (id, name) VALUES +(16, '$STANDARD_INFORMATION'), +(32, '$ATTRIBUTE_LIST'), +(48, '$FILE_NAME'), +(64, '$OBJECT_ID'), +(80, '$SECURITY_DESCRIPTOR'), +(96, '$VOLUME_NAME'), +(112, '$VOLUME_INFORMATION'), +(128, '$DATA'), +(144, '$INDEX_ROOT'), +(160, '$INDEX_ALLOCATION'), +(176, '$BITMAP'), +(192, '$REPARSE_POINT'), +(208, '$EA_INFORMATION'), +(224, '$EA'), +(256, '$LOGGED_UTILITY_STREAM'); \ No newline at end of file diff --git a/src/analyzeMFT/sql/file_record_flags.sql b/src/analyzeMFT/sql/file_record_flags.sql new file mode 100644 index 0000000..579f630 --- /dev/null +++ b/src/analyzeMFT/sql/file_record_flags.sql @@ -0,0 +1,10 @@ +CREATE TABLE file_record_flags ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL +); + +INSERT INTO file_record_flags (id, name) VALUES +(1, 'FILE_RECORD_IN_USE'), +(2, 'FILE_RECORD_IS_DIRECTORY'), +(4, 'FILE_RECORD_IS_EXTENSION'), +(8, 'FILE_RECORD_HAS_SPECIAL_INDEX'); \ No newline at end of file