commit or not commit

.gitattributes

@@ -0,0 +1,7 @@
+misc/assets/fonts/** filter=lfs diff=lfs merge=lfs -text
+public/json/tags_character.json filter=lfs diff=lfs merge=lfs -text
+plots/models_nsfw_monthly_histogram_with_quarters.png filter=lfs diff=lfs merge=lfs -text
+public/img/sankey.png filter=lfs diff=lfs merge=lfs -text
+public/img/tag_clusters.png filter=lfs diff=lfs merge=lfs -text
+public/img/training_tags.png filter=lfs diff=lfs merge=lfs -text
+plots/grid10x120.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,7 @@
+data
+ext
+ARCHIVE
+misc/api_keys.txt
+misc/training_tags_categories
+scripts/ARCHIVE
+scripts/CEMETARY

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ext/MiVOLO"]
+	path = ext/MiVOLO
+	url = https://github.com/WildChlamydia/MiVOLO.git

.virtual_documents/civitai_site_properties.ipynb

@@ -0,0 +1,148 @@
+import matplotlib.pyplot as plt
+from matplotlib.font_manager import FontProperties
+from matplotlib import font_manager
+from matplotlib.patches import Rectangle
+from matplotlib.patches import FancyBboxPatch
+import matplotlib.dates as mdates
+from dateutil import parser
+import json
+import requests
+import importlib
+import datetime
+import os
+from collections import Counter
+import functions
+import pandas as pd
+importlib.reload(functions)
+from functions import sortByFrequency, most_popular_tags, models_tags, get_model_info
+
+
+# Fonts
+font_manager.fontManager.addfont('fonts/Noto_Sans_JP/static/NotoSansJP-Medium.ttf')  # Fallback for Japanese
+font_manager.fontManager.addfont('fonts/Noto_Sans_SC/static/NotoSansSC-Medium.ttf')  # Fallback for Simplified Chinese
+plt.rcParams['font.family'] = ['Noto Sans JP', 'Noto Sans SC', 'sans-serif']
+
+
+
+get_download_urls('most_popular/most_popular_models_with_date_page_1.json')
+
+
+get_images()
+
+
+directory_path = "newest_images"  # Update this to your directory path
+march_items = []
+
+# Step 1: Filter items by March and collect browsingLevel
+for filename in os.listdir(directory_path):
+    if filename.endswith('.json'):
+        file_path = os.path.join(directory_path, filename)
+        with open(file_path, 'r', encoding='utf-8') as file:
+            data = json.load(file)
+            items = data['items']  # Access the list of items
+            for item in items:
+                created_at = datetime.fromisoformat(item['createdAt'][:-1])  # Remove 'Z' and convert to datetime
+                if created_at.month == 3 and created_at.year == 2024:  # Check if the item is from March 2024
+                    browsing_level = item.get('browsingLevel', None)
+                    if browsing_level in [2, 4, 8, 16]:  # Only include specified browsing levels
+                        march_items.append((created_at.day, browsing_level))
+
+# Prepare data for the histogram
+days = range(1, 32)  # Days in March
+levels = ['1 | PG', '2 | PG13', '4 | R' , '8 | X', '16 | XXX']
+colors = ['coral', 'lightgreen', 'salmon', 'blueviolet', 'blue']  # Different color for each browsingLevel
+labels = [f'Browsing Level: {level}' for level in levels]
+
+# Initialize a dictionary to hold day counts for each level
+day_counts_by_level = {level: [0] * 31 for level in levels}  # 31 days in March
+
+# Populate the day counts
+for day, level in march_items:
+    day_counts_by_level[level][day - 1] += 1  # Increment the count for the corresponding day and level
+
+# Plotting
+plt.figure(figsize=(10, 6))  # Optional: Adjust the figure size
+
+# Create a stacked bar chart
+bottom = [0] * 31
+for level, color, label in zip(levels, colors, labels):
+    plt.bar(days, day_counts_by_level[level], bottom=bottom, color=color, label=label, edgecolor='black', width=0.8)
+    bottom = [x + y for x, y in zip(bottom, day_counts_by_level[level])]
+
+# Set the title and labels
+plt.title('Number of Uploads per Day in March 2024 by Browsing Level')
+plt.xlabel('Day of the Month')
+plt.ylabel('Number of Uploads')
+plt.xticks(days)  # Set x-ticks to show every day of the month
+plt.legend()  # Show legend to identify browsingLevels
+
+# Show the plot
+plt.show()
+
+
+directory_path = 'most popular'  # Update this to your directory path
+hatch_pattern = '\\\\\\\\\\\\\\'
+# Initialize an empty list to store the type of each model
+model_types = []
+
+# Iterate over each file in the directory
+for filename in os.listdir(directory_path):
+    if filename.endswith('.json'):  # Check if the file is a JSON file
+        file_path = os.path.join(directory_path, filename)
+        with open(file_path, 'r') as file:
+            data = json.load(file)
+            items = data['items']
+            for item in items:
+                model_types.append(item['type'])
+
+# Convert the list of types into a pandas DataFrame
+df = pd.DataFrame(model_types, columns=['Type'])
+
+# Count the occurrences of each type
+type_counts = df['Type'].value_counts().reset_index()
+type_counts.columns = ['Type', 'Count']
+total = type_counts['Count'].sum()
+type_counts['Percentage'] = (type_counts['Count'] / total * 100).round(2)
+
+# Plotting
+plt.figure(figsize=(10, 5))
+bars = plt.barh(type_counts['Type'], type_counts['Count'], color='white', hatch=hatch_pattern, edgecolor='peachpuff')
+plt.ylabel('Model Type')
+plt.xlabel('Counts')
+
+ax = plt.gca()
+# Hide the spines but keep the tick labels
+ax.spines['top'].set_visible(False)
+ax.spines['right'].set_visible(False)
+ax.spines['left'].set_visible(False)
+ax.spines['bottom'].set_visible(False)
+
+ax.xaxis.set_ticks_position('none') 
+#ax.yaxis.set_ticks_position('none')
+
+plt.title('Model Type Distribution Across All Files')
+
+# Adding the percentages on the bars
+for bar, percentage in zip(bars, type_counts['Percentage']):
+    plt.text(bar.get_width()+ 100, bar.get_y() + bar.get_height()/2, f'{percentage}%', va='center', color='blueviolet')
+
+plt.tight_layout()  # Adjust layout to make room for the labels
+plt.show()
+
+
+
+
+
+## sortByFrequency function: path-to-json, skalar, title, output_path
+sortByFrequency('most popular/most_popular_models_with_date_page_1.json', 19000, 'Top 20 most downloaded models and their tags -', 'png/2024-03-26/most_downloaded')
+most_popular_tags('most_popular')
+#most_popular_tags_02('most_popular')
+
+
+importlib.reload(functions)
+
+
+most_popular_tags('most_popular')
+
+
+!pip install wordcloud matplotlib

misc/api_keys_example.txt

@@ -0,0 +1,2 @@
+api_key_01
+api_key_02

requirements.txt

@@ -0,0 +1,38 @@
+# requirements.txt
+
+# Core Python libraries
+numpy
+pandas
+matplotlib
+plotly
+scipy
+datetime
+dateutil
+pathlib
+itertools
+collections
+logging
+argparse
+shutil
+re
+json
+csv
+sys
+ast
+importlib
+struct
+
+# Machine Learning and Image Processing
+torch
+opencv-python
+pillow
+requests
+retrying
+mivolo
+
+# Jupyter-specific (if running in Jupyter)
+jupyterlab
+ipykernel
+
+# Additional Utilities
+fonttools

scripts/0_Scraping_image_metadata.ipynb

@@ -0,0 +1,1311 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "1111ea95-d385-49b9-a4d9-ef886ace5c7a",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T11:24:25.566747Z",
+     "iopub.status.busy": "2025-02-06T11:24:25.566066Z",
+     "iopub.status.idle": "2025-02-06T11:24:25.571748Z",
+     "shell.execute_reply": "2025-02-06T11:24:25.571305Z",
+     "shell.execute_reply.started": "2025-02-06T11:24:25.566705Z"
+    }
+   },
+   "source": [
+    "# 0 Scraping Metadata and Dataset consolidation\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6632505a-e7ca-4463-9ffc-e36fad42235f",
+   "metadata": {},
+   "source": [
+    "## IMAGES\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e3388bac-bb71-40bc-a693-9ac7a2d5f32c",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T10:08:22.229784Z",
+     "iopub.status.busy": "2025-02-06T10:08:22.229287Z",
+     "iopub.status.idle": "2025-02-06T10:08:22.232210Z",
+     "shell.execute_reply": "2025-02-06T10:08:22.231793Z",
+     "shell.execute_reply.started": "2025-02-06T10:08:22.229766Z"
+    }
+   },
+   "source": [
+    "### Step 1: Image metadata scraping, sorting and CSV consolidation"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "8b5d623a-a647-42ae-9951-efa99048cdf3",
+   "metadata": {},
+   "source": [
+    "Dataset creation workflow\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Scrape Data from API │ --> │ Use CivitAI API with a timestamp-based cursor to  │\n",
+    "│ Paginate Using Cursor│     │ scrape image metadata and save in JSON batches.   │\n",
+    "│ Save in JSON Format  │     └───────────────────────────────────────────────────┘\n",
+    "└─────────┬────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Locate JSON Files    │ --> │ Find all JSON files saved in the directory.       │\n",
+    "│ in Directory         │     │                                                   │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Read Each JSON File  │ --> │ Parse JSON files and extract metadata, reactions, │\n",
+    "│ Parse and Extract    │     │ and resource details.                             │\n",
+    "│ Metadata, Reactions, │     │                                                   │\n",
+    "│ and Resources        │     │                                                   │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Sort Items by        │ --> │ Sort the extracted items chronologically using    │\n",
+    "│ createdAt Timestamp  │     │ their createdAt timestamp.                        │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Write Extracted Data │ --> │ Save the processed and sorted data into a         │\n",
+    "│ to a Consolidated    │     │ consolidated CSV file with structured columns.    │\n",
+    "│ CSV File             │     │                                                   │\n",
+    "└────────��─────────────┘     └───────────────────────────────────────────────────┘\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "f8decb63-43f5-4731-823d-94632eee7618",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:37:51.115763Z",
+     "iopub.status.busy": "2025-02-08T19:37:51.114573Z",
+     "iopub.status.idle": "2025-02-08T19:37:51.170027Z",
+     "shell.execute_reply": "2025-02-08T19:37:51.169401Z",
+     "shell.execute_reply.started": "2025-02-08T19:37:51.115738Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "import csv\n",
+    "import requests\n",
+    "from datetime import datetime\n",
+    "import time\n",
+    "from pathlib import Path\n",
+    "import hashlib\n",
+    "import pandas as pd\n",
+    "import sys\n",
+    "from datetime import datetime, timedelta\n",
+    "import shutil"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4b2426c3-96a0-468e-b6dc-78dea9c3e92b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:37:51.809027Z",
+     "iopub.status.busy": "2025-02-08T19:37:51.808835Z",
+     "iopub.status.idle": "2025-02-08T19:37:51.812922Z",
+     "shell.execute_reply": "2025-02-08T19:37:51.812429Z",
+     "shell.execute_reply.started": "2025-02-08T19:37:51.809009Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "current_dir = Path.cwd()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "dd0daa43-d988-4ac5-b9c7-2f9c65078325",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-07T20:59:30.413939Z",
+     "iopub.status.busy": "2025-02-07T20:59:30.412861Z",
+     "iopub.status.idle": "2025-02-07T20:59:30.418660Z",
+     "shell.execute_reply": "2025-02-07T20:59:30.418054Z",
+     "shell.execute_reply.started": "2025-02-07T20:59:30.413918Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Define the input timestamp in ISO 8601 format\n",
+    "input_timestamp = \"2025-02-01T00:00:00.000Z\"  # point in time from when you want to obtain metadata (you can copy the timestamp from the last *.json batch obtained to get the data of longer timespans)\n",
+    "\n",
+    "# Function to convert an ISO 8601 date string to a Unix timestamp in milliseconds with a 2-hour offset \n",
+    "def iso_to_timestamp(iso_str):\n",
+    "    # Parse the ISO 8601 date string (including milliseconds and 'Z' indicating UTC)\n",
+    "    date = datetime.strptime(iso_str, \"%Y-%m-%dT%H:%M:%S.%fZ\") \n",
+    "    # Add 2 hours to the parsed date\n",
+    "    adjusted_date = date + timedelta(hours=2)\n",
+    "    # Convert to Unix timestamp (in milliseconds)\n",
+    "    timestamp = int(adjusted_date.timestamp() * 1000)\n",
+    "    return timestamp\n",
+    "\n",
+    "# Convert the input timestamp to an initial cursor\n",
+    "initial_cursor = iso_to_timestamp(input_timestamp)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "20b4bc72-80c4-4af6-bc76-aeecfa65f9dc",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-07T20:59:31.134620Z",
+     "iopub.status.busy": "2025-02-07T20:59:31.133548Z",
+     "iopub.status.idle": "2025-02-07T20:59:31.138081Z",
+     "shell.execute_reply": "2025-02-07T20:59:31.137555Z",
+     "shell.execute_reply.started": "2025-02-07T20:59:31.134599Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "data_raw = current_dir.parent / f\"data/raw/image_metadata/001/{input_timestamp.replace(':', '').replace('T', '_').replace('.', '_').replace('Z', '')}\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f69c921d-ca57-4746-8124-a0fa4fce80ff",
+   "metadata": {},
+   "source": [
+    "#### The *.json files will be saved in the following structure:"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "d731001e-32a4-4894-a2fe-835ea69214fc",
+   "metadata": {},
+   "source": [
+    "data/\n",
+    "└── raw/\n",
+    "    └── image_metadata\n",
+    "           └── 001/\n",
+    "                └── YYYYMMDD_HHMMSS/                 # Folder named based on the input timestamp\n",
+    "                    ├── most_recent_1.json           # Sequential JSON files\n",
+    "                    ├── most_recent_2.json\n",
+    "                    ├── ...\n",
+    "                    ├── most_recent_50000.json\n",
+    "                    └── YYYYMMDD_HHMMSS_session_TIMESTAMP/  # New folder after API limit is reached\n",
+    "                        ├── most_recent_50001.json\n",
+    "                        ├── most_recent_50002.json\n",
+    "                        ├── ...\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "4cb3c005-bf70-43fc-afe8-93bf30872fb4",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T16:51:57.402732Z",
+     "iopub.status.busy": "2025-02-06T16:51:57.402532Z",
+     "iopub.status.idle": "2025-02-06T16:52:00.855604Z",
+     "shell.execute_reply": "2025-02-06T16:52:00.854678Z",
+     "shell.execute_reply.started": "2025-02-06T16:51:57.402712Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "max_images = 50000 # some huge number now redundant because CivitAI API currently limits free api usage to 50'000\n",
+    "\n",
+    "def get_image_metadata():\n",
+    "    base_url = \"https://civitai.com/api/v1/images\"\n",
+    "    headers = {\n",
+    "        \"Accept\": \"application/json\",\n",
+    "        \"Authorization\": \"Bearer APITOKEN\"  # Replace with your actual API token\n",
+    "    }\n",
+    "    params = {\n",
+    "        \"sort\": \"Most Reactions\",\n",
+    "        \"nsfw\": \"X\",\n",
+    "        \"cursor\": f\"0|{initial_cursor}\"\n",
+    "    }\n",
+    "\n",
+    "    # Use pathlib to create the base directory for saving files\n",
+    "\n",
+    "    file_counter = 0\n",
+    "\n",
+    "    # Create a folder based on the input timestamp\n",
+    "    data_raw.mkdir(parents=True, exist_ok=True)\n",
+    "    sub_directory_path = data_raw\n",
+    "\n",
+    "    retry_delay = 300  # 300 seconds / 5 minutes\n",
+    "    retries_without_cursor = 0  # Track consecutive retries without a new cursor\n",
+    "\n",
+    "    while True:\n",
+    "        response = requests.get(base_url, headers=headers, params=params)\n",
+    "        if response.status_code == 200:\n",
+    "            data = response.json()\n",
+    "            items = data.get('items', [])\n",
+    "            if not items:\n",
+    "                print(\"No more data available.\")\n",
+    "                retries_without_cursor += 1\n",
+    "                if retries_without_cursor > 5:  # Allow up to 5 retries before stopping\n",
+    "                    print(\"Reached the end of the data after multiple retries.\")\n",
+    "                    break\n",
+    "                time.sleep(retry_delay)\n",
+    "                continue\n",
+    "\n",
+    "            retries_without_cursor = 0  # Reset if we get data\n",
+    "\n",
+    "            next_cursor = data.get('metadata', {}).get('nextCursor')\n",
+    "            if next_cursor:\n",
+    "                # Increment the cursor by 50 (e.g., \"0|1722470401000\" -> \"50|1722470401000\")\n",
+    "                cursor_value = int(params['cursor'].split(\"|\")[0])\n",
+    "                new_cursor_value = cursor_value + 50\n",
+    "                params['cursor'] = f\"{new_cursor_value}|{params['cursor'].split('|')[1]}\"\n",
+    "            else:\n",
+    "                print(\"No new cursor returned, stopping.\")\n",
+    "                break\n",
+    "\n",
+    "            file_counter += 1\n",
+    "            if file_counter % max_images == 0:\n",
+    "                time_stamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+    "                sub_directory_path = data_raw / f\"{input_timestamp.replace(':', '').replace('T', '_').replace('.', '_').replace('Z', '')}_session_{time_stamp}\"\n",
+    "                sub_directory_path.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "            file_path = sub_directory_path / f'most_recent_{file_counter}.json'\n",
+    "            with open(file_path, 'w', encoding='utf-8') as file:\n",
+    "                json.dump(data, file, indent=4)\n",
+    "\n",
+    "        elif response.status_code == 502:\n",
+    "            print(f\"Received HTTP 502. Retrying in {retry_delay // 60} minutes.\")\n",
+    "            time.sleep(retry_delay)  # Wait for 5 minutes before retrying\n",
+    "        else:\n",
+    "            print(f\"Failed to fetch data: HTTP {response.status_code}\")\n",
+    "            break\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "238044f1-544f-4821-a91e-57386d464f9e",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T16:13:44.748157Z",
+     "iopub.status.busy": "2025-02-06T16:13:44.747986Z",
+     "iopub.status.idle": "2025-02-06T16:13:44.752230Z",
+     "shell.execute_reply": "2025-02-06T16:13:44.751694Z",
+     "shell.execute_reply.started": "2025-02-06T16:13:44.748140Z"
+    }
+   },
+   "source": [
+    "uncomment this line to scrape image metadata:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "7c89cb68-983b-47df-a028-e02d7ca0829d",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T16:52:00.857210Z",
+     "iopub.status.busy": "2025-02-06T16:52:00.857015Z",
+     "iopub.status.idle": "2025-02-06T16:52:04.901096Z",
+     "shell.execute_reply": "2025-02-06T16:52:04.900159Z",
+     "shell.execute_reply.started": "2025-02-06T16:52:00.857191Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "#get_image_metadata()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "06f5f27e-7278-44c9-ae34-9a2020cac2f2",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "648aee1b-9e01-4400-a6a8-3361c97c377a",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T11:33:11.524990Z",
+     "iopub.status.busy": "2025-02-06T11:33:11.523287Z",
+     "iopub.status.idle": "2025-02-06T11:33:11.531385Z",
+     "shell.execute_reply": "2025-02-06T11:33:11.530884Z",
+     "shell.execute_reply.started": "2025-02-06T11:33:11.524893Z"
+    }
+   },
+   "source": [
+    "### Step 02 Chronological Sorting"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "b73d7ea5-50e0-485d-8bc5-a4736f31c789",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T16:52:04.902663Z",
+     "iopub.status.busy": "2025-02-06T16:52:04.902478Z",
+     "iopub.status.idle": "2025-02-06T16:52:08.491068Z",
+     "shell.execute_reply": "2025-02-06T16:52:08.490105Z",
+     "shell.execute_reply.started": "2025-02-06T16:52:04.902646Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def organize_files(source_dir, target_dir, max_items_per_file=100):\n",
+    "    print(f\"Starting to organize files from {source_dir} to {target_dir}\")\n",
+    "    item_buffer = []\n",
+    "    file_count = 0\n",
+    "\n",
+    "    # Walk through all files in the source directory\n",
+    "    for root, dirs, files in os.walk(source_dir):\n",
+    "        if '.ipynb_checkpoints' in root:\n",
+    "            continue  # Skip .ipynb_checkpoints directories\n",
+    "        print(f\"Checking directory: {root}\")\n",
+    "        for filename in files:\n",
+    "            if filename.lower().endswith('.json'):\n",
+    "                file_path = os.path.join(root, filename)\n",
+    "                try:\n",
+    "                    with open(file_path, 'r') as file:\n",
+    "                        data = json.load(file)\n",
+    "                        items = data.get('items', [])  # Get the list of items\n",
+    "                        for item in items:\n",
+    "                            item_buffer.append(item)\n",
+    "                            # Write out the buffer if it has reached the maximum size\n",
+    "                            if len(item_buffer) >= max_items_per_file:\n",
+    "                                write_items(item_buffer[:max_items_per_file], target_dir)\n",
+    "                                item_buffer = item_buffer[max_items_per_file:]\n",
+    "                                file_count += 1\n",
+    "                except json.JSONDecodeError:\n",
+    "                    print(f\"Error decoding JSON from file {file_path}\")\n",
+    "                except Exception as e:\n",
+    "                    print(f\"An error occurred with file {file_path}: {e}\")\n",
+    "\n",
+    "    # Write any remaining items in the buffer\n",
+    "    if item_buffer:\n",
+    "        write_items(item_buffer, target_dir)\n",
+    "        file_count += 1\n",
+    "\n",
+    "    #print(f\"Processed {file_count} files.\")\n",
+    "\n",
+    "def write_items(items, target_dir):\n",
+    "    # Use the createdAt from the first item to determine the directory\n",
+    "    created_at = items[0].get('createdAt')\n",
+    "    if created_at:\n",
+    "        date_obj = datetime.fromisoformat(created_at.rstrip(\"Z\"))\n",
+    "        new_dir = os.path.join(target_dir, f\"{date_obj.year}\", f\"{date_obj.year}-{date_obj.month:02d}\", f\"{date_obj.year}-{date_obj.month:02d}-{date_obj.day:02d}\")\n",
+    "        os.makedirs(new_dir, exist_ok=True)\n",
+    "        new_file_path = os.path.join(new_dir, f\"batch_{date_obj.strftime('%Y%m%dT%H%M%S')}.json\")\n",
+    "        with open(new_file_path, 'w', encoding='utf-8') as new_file:\n",
+    "            json.dump(items, new_file, indent=4)\n",
+    "        #print(f\"Wrote {len(items)} items to {new_file_path}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "07cebb8a-2955-4432-a0d1-7ee94975a34b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T16:52:08.492517Z",
+     "iopub.status.busy": "2025-02-06T16:52:08.492328Z",
+     "iopub.status.idle": "2025-02-06T16:53:13.648165Z",
+     "shell.execute_reply": "2025-02-06T16:53:13.647511Z",
+     "shell.execute_reply.started": "2025-02-06T16:52:08.492499Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Starting to organize files from /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/raw/image_metadata/001/2025-02-01_000000_000 to /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/sorted/image_metadata\n",
+      "Checking directory: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/raw/image_metadata/001/2025-02-01_000000_000\n"
+     ]
+    }
+   ],
+   "source": [
+    "data_sorted = current_dir.parent / 'data/sorted/image_metadata'\n",
+    "\n",
+    "origin = data_raw\n",
+    "target = data_sorted\n",
+    "\n",
+    "organize_files( origin, target)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9255fd24-54ee-4235-bc44-a0246e2fd927",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "64639464-6dc4-4997-a805-3f9f41f305c7",
+   "metadata": {},
+   "source": [
+    "### Step 3 CSV consolidation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "add5f97b-562a-4af5-a8f2-635433a8158f",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T12:03:15.512133Z",
+     "iopub.status.busy": "2025-02-06T12:03:15.511399Z",
+     "iopub.status.idle": "2025-02-06T12:03:15.516297Z",
+     "shell.execute_reply": "2025-02-06T12:03:15.515815Z",
+     "shell.execute_reply.started": "2025-02-06T12:03:15.512114Z"
+    }
+   },
+   "source": [
+    "#### this script walks through the directories in **image-metadata/** and creates the dataset as **.csv** \n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0f8c83aa-c143-4fac-b890-a8d4249067a0",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T12:02:29.925370Z",
+     "iopub.status.busy": "2025-02-06T12:02:29.923935Z",
+     "iopub.status.idle": "2025-02-06T12:02:31.421575Z",
+     "shell.execute_reply": "2025-02-06T12:02:31.420575Z",
+     "shell.execute_reply.started": "2025-02-06T12:02:29.925332Z"
+    }
+   },
+   "source": [
+    "\n",
+    "### Dataset Columns\n",
+    "\n",
+    "| **Column Name**       | **Description**                                                                                |\n",
+    "|------------------------|-----------------------------------------------------------------------------------------------|\n",
+    "| `createdAt`           | Timestamp when the item was created.                                                          |\n",
+    "| `url`                 | URL associated with the item.                                                                 |\n",
+    "| `positivePrompt`      | Positive prompts used in the generation process.                                              |\n",
+    "| `negativePrompt`      | Negative prompts used in the generation process.                                              |\n",
+    "| `nsfw`                | Indicates whether the item is NSFW (Not Safe for Work).                                       |\n",
+    "| `nsfwLevel`           | Level of NSFW content (e.g., Soft, Mature).                we only considered SFW!!           |\n",
+    "| `browsingLevel`       | Browsing level required to access the content.                                                |\n",
+    "| `statsSummary`        | All social reactions summed up: cryCount, likeCount, heartCount, CommentCount                 |\n",
+    "| `commentCount`        | Number of comments received.                                                                  |\n",
+    "| `username`            | Username of the creator of the item.                                                          |\n",
+    "| `Model`               | Model used to generate the content.                                                           |\n",
+    "| `Meta`                | Simplified meta details, including size, seed, steps, sampler, and version.                   |\n",
+    "| `VAE`                 | Variational Autoencoder (VAE) used, if any.                                                   |\n",
+    "| `resourceIDs`| Array of up to six resources (LoRA etc.), including name, type, and weight.                            |\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "f8db24f9-6678-4e7d-a3b7-453f33ad0f71",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-07T20:59:38.328530Z",
+     "iopub.status.busy": "2025-02-07T20:59:38.328049Z",
+     "iopub.status.idle": "2025-02-07T20:59:38.342970Z",
+     "shell.execute_reply": "2025-02-07T20:59:38.342400Z",
+     "shell.execute_reply.started": "2025-02-07T20:59:38.328508Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def find_json_files(directory):\n",
+    "    \"\"\"Walk through the directory and its subdirectories to find all JSON files.\"\"\"\n",
+    "    json_files = []\n",
+    "    for root, _, files in os.walk(directory):\n",
+    "        for file in files:\n",
+    "            if file.endswith('.json'):\n",
+    "                json_files.append(os.path.join(root, file))\n",
+    "    return json_files\n",
+    "\n",
+    "def hash_username(username):\n",
+    "    \"\"\"Convert a username into a unique hash.\"\"\"\n",
+    "    return hashlib.sha256(username.encode('utf-8')).hexdigest()[:16]  # Use first 16 characters for brevity\n",
+    "\n",
+    "def extract_resource_ids(meta, resources):\n",
+    "    \"\"\"Extract resource IDs from metadata and resources.\"\"\"\n",
+    "    resource_ids = set()  # Use a set to avoid duplicates\n",
+    "    hash_ids = set()  # Separate set to track hashes specifically\n",
+    "\n",
+    "    # Extract CivitAI model IDs\n",
+    "    if \"civitaiResources\" in meta:\n",
+    "        for resource in meta[\"civitaiResources\"]:\n",
+    "            if \"modelVersionId\" in resource:\n",
+    "                resource_ids.add(str(resource[\"modelVersionId\"]))  # Ensure all IDs are strings\n",
+    "\n",
+    "    # Extract model hashes\n",
+    "    if \"Model hash\" in meta:\n",
+    "        hash_ids.add(meta[\"Model hash\"])\n",
+    "    if \"base_model_hash\" in meta:\n",
+    "        hash_ids.add(meta[\"base_model_hash\"])\n",
+    "    if \"models\" in meta and isinstance(meta[\"models\"], list):\n",
+    "        hash_ids.update(str(model) for model in meta[\"models\"])\n",
+    "\n",
+    "    # Extract identifiers from resources list\n",
+    "    for resource in resources:\n",
+    "        if isinstance(resource, dict):  # Ensure resource is a dictionary\n",
+    "            resource_hash = resource.get(\"hash\")\n",
+    "            if resource_hash:\n",
+    "                hash_ids.add(str(resource_hash))  # Add to hash set\n",
+    "            else:\n",
+    "                name = resource.get(\"name\")\n",
+    "                if name:\n",
+    "                    resource_ids.add(str(name))\n",
+    "\n",
+    "    # Remove any named IDs if their hash exists\n",
+    "    resource_ids = {rid for rid in resource_ids if rid not in hash_ids}\n",
+    "\n",
+    "    # Combine hashes and remaining resource IDs\n",
+    "    final_ids = list(hash_ids) + list(resource_ids)\n",
+    "    return final_ids\n",
+    "\n",
+    "def format_resource_ids(resource_ids):\n",
+    "    \"\"\"Format resource IDs in a standardized list format.\"\"\"\n",
+    "    formatted_ids = [str(rid).strip() for rid in resource_ids if rid]\n",
+    "    return f\"[{', '.join(formatted_ids)}]\"\n",
+    "\n",
+    "def truncate_prompt(prompt, max_tokens=77):\n",
+    "    \"\"\"Truncate a prompt to a maximum number of tokens.\"\"\"\n",
+    "    return ' '.join(prompt.split()[:max_tokens]) if prompt else \"\"\n",
+    "\n",
+    "def write_to_csv(json_files, output_csv):\n",
+    "    \"\"\"Read JSON files, extract data, and write to a CSV file.\"\"\"\n",
+    "    with open(output_csv, 'w', newline='', encoding='utf-8') as csvfile:\n",
+    "        fieldnames = [\n",
+    "            'id', 'createdAt', 'url', 'positivePrompt', 'negativePrompt', 'nsfw',\n",
+    "            'browsingLevel', 'statsSummary', 'usernameHash', 'Model', 'cfgScale',\n",
+    "            'sampler', 'Size', 'seed', 'VAE', 'generationSystem', 'resourceIDs'\n",
+    "        ]\n",
+    "        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
+    "        writer.writeheader()\n",
+    "\n",
+    "        for json_file in json_files:\n",
+    "            with open(json_file, 'r', encoding='utf-8') as file:\n",
+    "                try:\n",
+    "                    data = json.load(file)\n",
+    "                    for item in data:\n",
+    "                        meta = item.get('meta', {}) or {}\n",
+    "                        stats = item.get('stats', {}) or {}\n",
+    "                        resources = meta.get('resources', []) if isinstance(meta, dict) else []\n",
+    "\n",
+    "                        # Anonymize username\n",
+    "                        username = item.get('username', '')\n",
+    "                        username_hash = hash_username(username) if username else ''\n",
+    "\n",
+    "                        # Extract resource IDs\n",
+    "                        resource_ids = extract_resource_ids(meta, resources)\n",
+    "                        formatted_resource_ids = format_resource_ids(resource_ids)\n",
+    "\n",
+    "                        # Summarize stats counts\n",
+    "                        stats_summary = sum(stats.get(key, 0) for key in stats)\n",
+    "\n",
+    "                        # Truncate prompts\n",
+    "                        positive_prompt = truncate_prompt(meta.get('prompt', ''))\n",
+    "                        negative_prompt = truncate_prompt(meta.get('negativePrompt', ''))\n",
+    "\n",
+    "                        # Extract metadata fields\n",
+    "                        cfg_scale = meta.get('cfgScale', 'N/A')\n",
+    "                        sampler = meta.get('sampler', 'N/A')\n",
+    "                        size = meta.get('Size', 'N/A')\n",
+    "                        seed = meta.get('seed', 'N/A')\n",
+    "                        vae = meta.get('VAE', 'N/A')\n",
+    "                        generation_system = \"CivitAI\" if \"civitaiResources\" in meta else \"Undetermined\"\n",
+    "\n",
+    "                        # Process a single row\n",
+    "                        row = {\n",
+    "                            'id': item.get('id', ''),\n",
+    "                            'createdAt': item.get('createdAt', ''),\n",
+    "                            'url': item.get('url', ''),\n",
+    "                            'positivePrompt': positive_prompt,\n",
+    "                            'negativePrompt': negative_prompt,\n",
+    "                            'nsfw': item.get('nsfw', False),\n",
+    "                            'browsingLevel': item.get('browsingLevel', 'N/A'),\n",
+    "                            'statsSummary': stats_summary,\n",
+    "                            'usernameHash': username_hash,\n",
+    "                            'Model': meta.get('Model', ''),\n",
+    "                            'cfgScale': cfg_scale,\n",
+    "                            'sampler': sampler,\n",
+    "                            'Size': size,\n",
+    "                            'seed': seed,\n",
+    "                            'VAE': vae,\n",
+    "                            'generationSystem': generation_system,\n",
+    "                            'resourceIDs': formatted_resource_ids\n",
+    "                        }\n",
+    "\n",
+    "                        writer.writerow(row)\n",
+    "                except (json.JSONDecodeError, KeyError, TypeError) as e:\n",
+    "                    print(f\"Error processing file {json_file}: {e}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1c51189e-5eb2-40ef-a5dd-807cd9cded5f",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T12:49:11.129158Z",
+     "iopub.status.busy": "2025-02-06T12:49:11.128195Z",
+     "iopub.status.idle": "2025-02-06T12:49:11.132440Z",
+     "shell.execute_reply": "2025-02-06T12:49:11.131779Z",
+     "shell.execute_reply.started": "2025-02-06T12:49:11.129132Z"
+    }
+   },
+   "source": [
+    "### Save as 'Civiverse' dataset in data/CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "07c97ff9-7ce0-43b7-a156-f5fb79c49cd9",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-07T20:59:40.218975Z",
+     "iopub.status.busy": "2025-02-07T20:59:40.218224Z",
+     "iopub.status.idle": "2025-02-07T21:04:11.221872Z",
+     "shell.execute_reply": "2025-02-07T21:04:11.220974Z",
+     "shell.execute_reply.started": "2025-02-07T20:59:40.218926Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error processing file /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/sorted/image_metadata/2024/2024-07/2024-07-29/batch_20240729T192800.json: Expecting value: line 1 column 1 (char 0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "raw = current_dir.parent / 'data/sorted/image_metadata/'\n",
+    "csv_output = current_dir.parent / 'data/CSV/Civiverse.csv'\n",
+    "\n",
+    "csv_output.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "json_files = find_json_files(raw)\n",
+    "write_to_csv(json_files, csv_output)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b3e64b07-98d1-480d-b4e1-f624a17db848",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T13:50:08.120671Z",
+     "iopub.status.busy": "2025-02-06T13:50:08.119949Z",
+     "iopub.status.idle": "2025-02-06T13:50:08.124761Z",
+     "shell.execute_reply": "2025-02-06T13:50:08.124298Z",
+     "shell.execute_reply.started": "2025-02-06T13:50:08.120632Z"
+    }
+   },
+   "source": [
+    "### Optional step 4: Create sampled subset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "b531b8cd-4653-451d-a6d5-386a2721827c",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T16:55:01.916121Z",
+     "iopub.status.busy": "2025-02-06T16:55:01.915489Z",
+     "iopub.status.idle": "2025-02-06T16:55:01.920043Z",
+     "shell.execute_reply": "2025-02-06T16:55:01.919655Z",
+     "shell.execute_reply.started": "2025-02-06T16:55:01.916103Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "input_file = current_dir.parent / 'data/CSV/Civiverse.csv'\n",
+    "output_file = current_dir.parent / 'data/CSV/image_subsets/Civiverse-mini.csv'\n",
+    "output_file.parent.mkdir(parents=True, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "b82db8b3-310c-49e4-9d7a-aa34662f9556",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T16:55:01.920861Z",
+     "iopub.status.busy": "2025-02-06T16:55:01.920586Z",
+     "iopub.status.idle": "2025-02-06T16:55:06.790164Z",
+     "shell.execute_reply": "2025-02-06T16:55:06.789503Z",
+     "shell.execute_reply.started": "2025-02-06T16:55:01.920844Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/sctmp/lauwag/ipykernel_1043676/3764041865.py:4: DtypeWarning: Columns (13) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  df = pd.read_csv(input_file)\n",
+      "/sctmp/lauwag/ipykernel_1043676/3764041865.py:10: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  df_sample['createdAt'] = pd.to_datetime(df_sample['createdAt'], errors='coerce')\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "# Define the sampling rate\n",
+    "n = 100\n",
+    "# Read the CSV file\n",
+    "df = pd.read_csv(input_file)\n",
+    "\n",
+    "# Sample every 15th row\n",
+    "df_sample = df.iloc[::n, :]\n",
+    "\n",
+    "# Convert the 'createdAt' column to datetime, inferring mixed formats\n",
+    "df_sample['createdAt'] = pd.to_datetime(df_sample['createdAt'], errors='coerce')\n",
+    "\n",
+    "# Sort the sampled data by the 'createdAt' column\n",
+    "df_sample_sorted = df_sample.sort_values(by='createdAt')\n",
+    "\n",
+    "# Save the sorted sample DataFrame to a new CSV file\n",
+    "df_sample_sorted.to_csv(output_file, index=False)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d915b7d6-937a-4fa9-9edc-ded67e3606f5",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "---\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "11647bb7-5ce9-414a-8486-5bdce8d9cfea",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T12:49:43.126762Z",
+     "iopub.status.busy": "2025-02-06T12:49:43.125797Z",
+     "iopub.status.idle": "2025-02-06T12:49:43.129759Z",
+     "shell.execute_reply": "2025-02-06T12:49:43.129176Z",
+     "shell.execute_reply.started": "2025-02-06T12:49:43.126736Z"
+    }
+   },
+   "source": [
+    "## MODELS"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5c9cb5e3-7cea-4574-9319-f3cd89354b1f",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T13:23:38.017078Z",
+     "iopub.status.busy": "2025-02-06T13:23:38.016639Z",
+     "iopub.status.idle": "2025-02-06T13:23:38.019993Z",
+     "shell.execute_reply": "2025-02-06T13:23:38.019549Z",
+     "shell.execute_reply.started": "2025-02-06T13:23:38.017053Z"
+    }
+   },
+   "source": [
+    "### Step 1: Scrape model metadata"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "83e12b9f-5ae1-407d-9754-5979d837f787",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T14:06:04.857874Z",
+     "iopub.status.busy": "2025-02-06T14:06:04.857500Z",
+     "iopub.status.idle": "2025-02-06T14:06:04.860438Z",
+     "shell.execute_reply": "2025-02-06T14:06:04.860030Z",
+     "shell.execute_reply.started": "2025-02-06T14:06:04.857856Z"
+    }
+   },
+   "source": [
+    "#### the resulting files will appear in data/raw/model_metadata as *.json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "41ee14e0-fb78-4f91-aba1-13faf05af7d8",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:37:56.687832Z",
+     "iopub.status.busy": "2025-02-08T19:37:56.687251Z",
+     "iopub.status.idle": "2025-02-08T19:37:56.696572Z",
+     "shell.execute_reply": "2025-02-08T19:37:56.696059Z",
+     "shell.execute_reply.started": "2025-02-08T19:37:56.687809Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import datetime\n",
+    "\n",
+    "def load_api_keys():\n",
+    "    \"\"\"Load API keys from a text file, one per line.\"\"\"\n",
+    "    if not os.path.exists(key_karussell):\n",
+    "        raise FileNotFoundError(f\"API key file '{API_KEYS_FILE}' not found!\")\n",
+    "    \n",
+    "    with open(key_karussell, 'r') as file:\n",
+    "        keys = [line.strip() for line in file if line.strip()]\n",
+    "    \n",
+    "    if not keys:\n",
+    "        raise ValueError(\"No API keys found in the file!\")\n",
+    "    \n",
+    "    return keys\n",
+    "\n",
+    "def get_model_metadata():\n",
+    "    base_url = \"https://civitai.com/api/v1/models\"\n",
+    "    params = {\"sort\": \"Newest\", \"nsfw\": True}\n",
+    "\n",
+    "    # Load API keys\n",
+    "    api_keys = load_api_keys()\n",
+    "    key_index = 0  # Start with the first key\n",
+    "\n",
+    "    page_counter = 0\n",
+    "    max_pages = 300000000  # Adjust as needed\n",
+    "    os.makedirs(directory_path, exist_ok=True)\n",
+    "\n",
+    "    while True:\n",
+    "        if page_counter >= max_pages:\n",
+    "            print(f\"Reached the limit of {max_pages} pages.\")\n",
+    "            break\n",
+    "\n",
+    "        headers = {\n",
+    "            \"Accept\": \"application/json\",\n",
+    "            \"Authorization\": f\"Bearer {api_keys[key_index]}\"\n",
+    "        }\n",
+    "\n",
+    "        response = requests.get(base_url, headers=headers, params=params)\n",
+    "\n",
+    "        if response.status_code == 200:\n",
+    "            data = response.json()\n",
+    "            page_counter += 1\n",
+    "\n",
+    "            # Add timestamp\n",
+    "            formatted_timestamp = datetime.datetime.now().strftime(\"data obtained on the %d.%m.%Y at %H:%M CEST\")\n",
+    "\n",
+    "            data['timestamp'] = formatted_timestamp\n",
+    "\n",
+    "            # Save data to file\n",
+    "            file_path = os.path.join(directory_path, f'newest_models_{page_counter}.json')\n",
+    "            with open(file_path, 'w', encoding='utf-8') as file:\n",
+    "                json.dump(data, file, indent=4)\n",
+    "\n",
+    "            # Check for nextCursor\n",
+    "            next_cursor = data.get('metadata', {}).get('nextCursor')\n",
+    "            if not next_cursor:\n",
+    "                print(\"No more data available.\")\n",
+    "                break\n",
+    "            else:\n",
+    "                params['cursor'] = next_cursor\n",
+    "        \n",
+    "        elif response.status_code in (401, 403):  # Unauthorized or Forbidden\n",
+    "            print(f\"API Key {key_index + 1} failed with status {response.status_code}. Trying next key...\")\n",
+    "            key_index += 1\n",
+    "\n",
+    "            if key_index >= len(api_keys):\n",
+    "                print(\"All API keys failed. Exiting.\")\n",
+    "                break  # Stop if all keys fail\n",
+    "        \n",
+    "        else:\n",
+    "            print(f\"Failed to fetch data: HTTP {response.status_code}\")\n",
+    "            break  # Stop on other errors\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0b574bc0-8bc5-48fd-9ced-8723d07e0d0e",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:37:57.395918Z",
+     "iopub.status.busy": "2025-02-08T19:37:57.395733Z",
+     "iopub.status.idle": "2025-02-08T19:37:57.399679Z",
+     "shell.execute_reply": "2025-02-08T19:37:57.399183Z",
+     "shell.execute_reply.started": "2025-02-08T19:37:57.395902Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "key_karussell = current_dir.parent / 'misc/api_keys.txt'\n",
+    "directory_path = current_dir.parent / 'data/raw/model_metadata/'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8f43ce23-5986-4b67-be2d-453831af9a6e",
+   "metadata": {},
+   "source": [
+    "uncomment this to get model metadata"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "3f95b4ba-5742-4268-b2e8-de9145faf495",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:37:58.117386Z",
+     "iopub.status.busy": "2025-02-08T19:37:58.117158Z",
+     "iopub.status.idle": "2025-02-08T19:37:58.121162Z",
+     "shell.execute_reply": "2025-02-08T19:37:58.120580Z",
+     "shell.execute_reply.started": "2025-02-08T19:37:58.117369Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "#get_model_metadata()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1ed5f44e-612e-4b6d-8974-904fb3e058d6",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T12:52:40.162173Z",
+     "iopub.status.busy": "2025-02-06T12:52:40.159989Z",
+     "iopub.status.idle": "2025-02-06T12:52:40.170634Z",
+     "shell.execute_reply": "2025-02-06T12:52:40.169945Z",
+     "shell.execute_reply.started": "2025-02-06T12:52:40.162124Z"
+    }
+   },
+   "source": [
+    "### Step 2 Consolidate Model-dataset CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "464d82f5-c24e-4b53-9b50-682fa4bf3430",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:37:59.052245Z",
+     "iopub.status.busy": "2025-02-08T19:37:59.051645Z",
+     "iopub.status.idle": "2025-02-08T19:37:59.056017Z",
+     "shell.execute_reply": "2025-02-08T19:37:59.055598Z",
+     "shell.execute_reply.started": "2025-02-08T19:37:59.052224Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "## path thingy\n",
+    "try: #scripts\n",
+    "    current_dir = Path(__file__).resolve().parent\n",
+    "except NameError:\n",
+    "    # jupyter\n",
+    "    current_dir = Path.cwd()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "14f3db4d-ef93-4629-8a27-971adb49248b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:37:59.443457Z",
+     "iopub.status.busy": "2025-02-08T19:37:59.443236Z",
+     "iopub.status.idle": "2025-02-08T19:37:59.890103Z",
+     "shell.execute_reply": "2025-02-08T19:37:59.889571Z",
+     "shell.execute_reply.started": "2025-02-08T19:37:59.443439Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing file: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/raw/model_metadata/newest_models_2.json\n",
+      "Processing file: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/raw/model_metadata/newest_models_1.json\n",
+      "Processing file: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/raw/model_metadata/newest_models_4.json\n",
+      "Processing file: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/raw/model_metadata/newest_models_6.json\n",
+      "Processing file: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/raw/model_metadata/newest_models_7.json\n",
+      "Processing file: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/raw/model_metadata/newest_models_5.json\n",
+      "Processing file: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/raw/model_metadata/newest_models_3.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "import pandas as pd\n",
+    "import hashlib\n",
+    "\n",
+    "\n",
+    "json_path = current_dir.parent / 'data/raw/model_metadata/'\n",
+    "\n",
+    "# Initialize a list to hold all the processed records\n",
+    "data_records = []\n",
+    "\n",
+    "def hash_username(username):\n",
+    "    \"\"\"Convert a username into a unique hash.\"\"\"\n",
+    "    return hashlib.sha256(username.encode('utf-8')).hexdigest()[:16]  # Use first 16 characters for brevity\n",
+    "\n",
+    "# Loop through each file in the directory\n",
+    "for filename in os.listdir(json_path):\n",
+    "    # Construct the full path to the JSON file\n",
+    "    file_path = os.path.join(json_path, filename)\n",
+    "\n",
+    "    # Only process files with .json extension\n",
+    "    if filename.endswith('.json') and os.path.isfile(file_path):\n",
+    "        print(f\"Processing file: {file_path}\")\n",
+    "\n",
+    "        # Open and load the JSON file\n",
+    "        with open(file_path, 'r') as file:\n",
+    "            try:\n",
+    "                data = json.load(file)\n",
+    "            except json.JSONDecodeError as e:\n",
+    "                print(f\"Error decoding JSON in file {filename}: {e}\")\n",
+    "                continue\n",
+    "\n",
+    "            # Determine how the items are structured within the dictionary\n",
+    "            if 'items' in data:\n",
+    "                items = data['items']\n",
+    "            elif 'data' in data:\n",
+    "                items = data['data']\n",
+    "            else:\n",
+    "                print(f\"No known item list key found in {filename}\")\n",
+    "                continue\n",
+    "\n",
+    "            for item in items:\n",
+    "                if not isinstance(item, dict):\n",
+    "                    print(f\"Expected a dictionary for each item, but found: {type(item)} in {filename}\")\n",
+    "                    continue\n",
+    "\n",
+    "                # Extract required information\n",
+    "                model_versions = item.get('modelVersions', [])\n",
+    "                download_url = model_versions[0]['files'][0]['downloadUrl'] if model_versions and 'files' in model_versions[0] and model_versions[0]['files'] else ''\n",
+    "                auto_hashes = model_versions[0]['files'][0]['hashes'] if model_versions and 'files' in model_versions[0] and model_versions[0]['files'] else {}\n",
+    "\n",
+    "                # Collect up to 20 modelVersion IDs\n",
+    "                model_version_ids = [mv.get('id', '') for mv in model_versions[:20]]\n",
+    "\n",
+    "                # Get preview images: first and latest\n",
+    "                first_image_url = model_versions[0]['images'][0]['url'] if model_versions and 'images' in model_versions[0] and model_versions[0]['images'] else ''\n",
+    "                latest_index = len(model_versions) - 1\n",
+    "                latest_image_url = model_versions[latest_index]['images'][0]['url'] if model_versions and 'images' in model_versions[latest_index] and model_versions[latest_index]['images'] else ''\n",
+    "\n",
+    "                # Hash the username\n",
+    "                username = item.get('creator', {}).get('username', '')\n",
+    "                username_hash = hash_username(username) if username else ''\n",
+    "\n",
+    "                # Extract additional fields\n",
+    "                record = {\n",
+    "                    'id': item.get('id', ''),\n",
+    "                    'name': item.get('name', ''),\n",
+    "                    'type': item.get('type', ''),\n",
+    "                    'baseModel': model_versions[0].get('baseModel', '') if model_versions else '',\n",
+    "                    'downloadCount': item.get('stats', {}).get('downloadCount', 0),\n",
+    "                    'nsfwLevel': item.get('nsfwLevel', 0),\n",
+    "                    'modelVersions': len(model_versions),\n",
+    "                    'publishedAt': model_versions[0].get('publishedAt', '') if model_versions else '',\n",
+    "                    'usernameHash': username_hash,\n",
+    "                    'downloadUrl': download_url,\n",
+    "                    'firstImageUrl': first_image_url,\n",
+    "                    'latestImageUrl': latest_image_url,\n",
+    "                    'poi': item.get('poi', False),\n",
+    "                    'AutoV1': auto_hashes.get('AutoV1', ''),\n",
+    "                    'AutoV2': auto_hashes.get('AutoV2', ''),\n",
+    "                    'AutoV3': auto_hashes.get('AutoV3', ''),\n",
+    "                    'SHA256': auto_hashes.get('SHA256', ''),\n",
+    "                    'CRC32': auto_hashes.get('CRC32', ''),\n",
+    "                    'BLAKE3': auto_hashes.get('BLAKE3', ''),\n",
+    "                    'previewImage': latest_image_url\n",
+    "                }\n",
+    "\n",
+    "                # Add version IDs to the record\n",
+    "                for i in range(20):  # Collect up to 20 version IDs\n",
+    "                    version_key = f'version_id_{i+1}'\n",
+    "                    record[version_key] = model_version_ids[i] if len(model_version_ids) > i else ''\n",
+    "\n",
+    "                # Add tags\n",
+    "                tags = item.get('tags', [])\n",
+    "                for i in range(7):  # To retrieve up to tag 6\n",
+    "                    tag_key = f'tag_{i+1}'\n",
+    "                    record[tag_key] = tags[i] if len(tags) > i else ''\n",
+    "\n",
+    "                data_records.append(record)\n",
+    "\n",
+    "# Create a DataFrame and sort by download count\n",
+    "df = pd.DataFrame(data_records)\n",
+    "df_sorted = df.sort_values(by='downloadCount', ascending=False)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2d6f822f-754f-4feb-9ed6-6f868421c454",
+   "metadata": {},
+   "source": [
+    "### Save model-data to CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "f861cf7b-5eb3-46ad-ab2d-9e2fdf2169b4",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:38:02.047421Z",
+     "iopub.status.busy": "2025-02-08T19:38:02.046761Z",
+     "iopub.status.idle": "2025-02-08T19:38:02.193381Z",
+     "shell.execute_reply": "2025-02-08T19:38:02.192886Z",
+     "shell.execute_reply.started": "2025-02-08T19:38:02.047397Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Data has been saved to /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/Civiverse-Models.csv\n"
+     ]
+    }
+   ],
+   "source": [
+    "output_csv = current_dir.parent / 'data/CSV/Civiverse-Models.csv'\n",
+    "output_csv.parent.mkdir(parents=True, exist_ok=True)\n",
+    "df_sorted.to_csv(output_csv, index=False)\n",
+    "print(f\"Data has been saved to {output_csv}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9b948933-2a2f-42b0-9083-2d81586ae3f2",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T13:42:32.447860Z",
+     "iopub.status.busy": "2025-02-06T13:42:32.446832Z",
+     "iopub.status.idle": "2025-02-06T13:42:32.453827Z",
+     "shell.execute_reply": "2025-02-06T13:42:32.453271Z",
+     "shell.execute_reply.started": "2025-02-06T13:42:32.447821Z"
+    }
+   },
+   "source": [
+    "### Step 3 Create Subsets: Checkpoint only, POI True, POI False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "ae894e26-4984-40e6-80a6-54fb6b61c873",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:38:03.089160Z",
+     "iopub.status.busy": "2025-02-08T19:38:03.088490Z",
+     "iopub.status.idle": "2025-02-08T19:38:03.093067Z",
+     "shell.execute_reply": "2025-02-08T19:38:03.092634Z",
+     "shell.execute_reply.started": "2025-02-08T19:38:03.089138Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "file_path = current_dir.parent / 'data/CSV/Civiverse-Models.csv'  # Update this with your actual file path\n",
+    "(current_dir.parent / 'data/CSV/model_subsets').mkdir(parents=True, exist_ok=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "88438f88-c723-423e-a4e5-e07f59096b72",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T19:41:39.279495Z",
+     "iopub.status.busy": "2025-02-08T19:41:39.279039Z",
+     "iopub.status.idle": "2025-02-08T19:41:39.674445Z",
+     "shell.execute_reply": "2025-02-08T19:41:39.673981Z",
+     "shell.execute_reply.started": "2025-02-08T19:41:39.279476Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Files saved successfully!\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Load the dataset\n",
+    "\n",
+    "data = pd.read_csv(file_path)\n",
+    "\n",
+    "# Version 1: Only 'poi' true models\n",
+    "poi_true_models = data[data['poi'] == True]\n",
+    "\n",
+    "# Version 2: Only types lora, dora, locon, textual inversion\n",
+    "specific_types = ['LORA', 'DORA', 'LOCON', 'textualInversion']\n",
+    "adapters = data[data['type'].isin(specific_types)]\n",
+    "\n",
+    "# Version 3: Only type checkpoint\n",
+    "checkpoint_models = data[data['type'] == 'Checkpoint']\n",
+    "\n",
+    "# Version 4: All models apart from 'poi' true\n",
+    "non_poi_models = data[data['poi'] != True]\n",
+    "\n",
+    "# Version 5: All models apart from 'poi' true and with nsfwLevel below 13\n",
+    "non_poi_low_nsfw_models = data[(data['poi'] != True) & (data['nsfwLevel'] < 13)]\n",
+    "\n",
+    "# Save the versions as separate CSV files\n",
+    "poi_true_models.to_csv(current_dir.parent / 'data/CSV/model_subsets/Civiverse_adapters_poi_true.csv', index=False)\n",
+    "adapters.to_csv(current_dir.parent / 'data/CSV/adapters.csv', index=False)\n",
+    "checkpoint_models.to_csv(current_dir.parent / 'data/CSV/model_subsets/Civiverse_checkpoint_only.csv', index=False)\n",
+    "non_poi_models.to_csv(current_dir.parent / 'data/CSV/model_subsets/Civiverse_adapters_poi_false.csv', index=False)\n",
+    "\n",
+    "print(\"Files saved successfully!\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "49e35088-8b2e-4189-83e1-3098d55dcad2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

scripts/Section_2_image_grid.ipynb

@@ -0,0 +1,389 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e8752a29-06b6-49a2-a724-a7b1c8f3d23b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T14:16:50.014780Z",
+     "iopub.status.busy": "2025-02-06T14:16:50.013936Z",
+     "iopub.status.idle": "2025-02-06T14:16:50.018030Z",
+     "shell.execute_reply": "2025-02-06T14:16:50.017506Z",
+     "shell.execute_reply.started": "2025-02-06T14:16:50.014757Z"
+    }
+   },
+   "source": [
+    "# Section 2. Download Images and create image Grid"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6f81ff2d-7763-4ac5-a9e3-d9a74541e143",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T16:47:42.502304Z",
+     "iopub.status.busy": "2025-02-06T16:47:42.501200Z",
+     "iopub.status.idle": "2025-02-06T16:47:42.746600Z",
+     "shell.execute_reply": "2025-02-06T16:47:42.745736Z",
+     "shell.execute_reply.started": "2025-02-06T16:47:42.502269Z"
+    }
+   },
+   "source": [
+    "This script downloads images and places them under data/sorted/YYYY/YYYY-MM/YYYY-MM-DD. A grid is created that blurs all NSFW true images and assigns a color overlay based on the color-coding of the BrowsingLevel"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d6d7d826-353c-4c1b-9d69-fa67f07e5245",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T16:17:00.985310Z",
+     "iopub.status.busy": "2025-02-06T16:17:00.984540Z",
+     "iopub.status.idle": "2025-02-06T16:17:01.124348Z",
+     "shell.execute_reply": "2025-02-06T16:17:01.123740Z",
+     "shell.execute_reply.started": "2025-02-06T16:17:00.985282Z"
+    }
+   },
+   "source": [
+    "![Alt text](../plots/grid2x24.png)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "6e5227f5-925c-4155-8015-53045794b986",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:58:48.202108Z",
+     "iopub.status.busy": "2025-02-08T21:58:48.201629Z",
+     "iopub.status.idle": "2025-02-08T21:58:48.485464Z",
+     "shell.execute_reply": "2025-02-08T21:58:48.484783Z",
+     "shell.execute_reply.started": "2025-02-08T21:58:48.202077Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from PIL import Image, ImageFilter, ImageDraw\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "import json\n",
+    "import matplotlib.colors as mcolors\n",
+    "import os\n",
+    "import json\n",
+    "import requests\n",
+    "from datetime import datetime\n",
+    "import argparse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "25cb523d-6d22-4d13-84f1-f9e38534d08e",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:58:49.153040Z",
+     "iopub.status.busy": "2025-02-08T21:58:49.152592Z",
+     "iopub.status.idle": "2025-02-08T21:58:49.156982Z",
+     "shell.execute_reply": "2025-02-08T21:58:49.156561Z",
+     "shell.execute_reply.started": "2025-02-08T21:58:49.153017Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "current_dir = Path.cwd()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "037ecae3-a060-4d90-beaf-040dfb018696",
+   "metadata": {},
+   "source": [
+    "## Step 1: Download images "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "948809b0-e78a-4821-86d0-fe4eb156265a",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:58:50.811934Z",
+     "iopub.status.busy": "2025-02-08T21:58:50.811459Z",
+     "iopub.status.idle": "2025-02-08T21:58:50.823112Z",
+     "shell.execute_reply": "2025-02-08T21:58:50.822600Z",
+     "shell.execute_reply.started": "2025-02-08T21:58:50.811911Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "import requests\n",
+    "from datetime import datetime\n",
+    "from PIL import Image\n",
+    "\n",
+    "def download_and_save_data(input_dir, output_dir):\n",
+    "    print(f\"Scanning directory: {input_dir}\")\n",
+    "    found_json = False\n",
+    "    for root, dirs, files in os.walk(input_dir):\n",
+    "        for file in files:\n",
+    "            if file.endswith('.json'):\n",
+    "                found_json = True\n",
+    "                file_path = os.path.join(root, file)\n",
+    "                print(f\"Processing JSON file: {file_path}\")\n",
+    "                try:\n",
+    "                    with open(file_path, 'r', encoding='utf-8') as json_file:\n",
+    "                        items = json.load(json_file)\n",
+    "                        for item in items:\n",
+    "                            if isinstance(item, dict):\n",
+    "                                process_item(item, root, output_dir, input_dir)\n",
+    "                except json.JSONDecodeError:\n",
+    "                    print(f\"Error decoding JSON from file {file_path}\")\n",
+    "                except Exception as e:\n",
+    "                    print(f\"An error occurred with file {file_path}: {e}\")\n",
+    "    if not found_json:\n",
+    "        print(\"No JSON files found in the directory.\")\n",
+    "\n",
+    "def process_item(item, root, output_dir, input_dir):\n",
+    "    if 'createdAt' in item and 'url' in item:\n",
+    "        created_at = datetime.strptime(item['createdAt'], \"%Y-%m-%dT%H:%M:%S.%fZ\")\n",
+    "        time_str = created_at.strftime(\"%Y%m%dT%H%M%S%f\")  # Include microseconds in the filename\n",
+    "        relative_path = os.path.relpath(root, input_dir)\n",
+    "        save_path = os.path.join(output_dir, relative_path)\n",
+    "        os.makedirs(save_path, exist_ok=True)\n",
+    "        image_path = os.path.join(save_path, f\"{time_str}.jpeg\")\n",
+    "        download_image(item['url'], image_path)\n",
+    "        resize_image(image_path, 512)  # Resize if necessary\n",
+    "        save_prompt(item.get('meta', {}).get('prompt', 'No prompt available'), os.path.join(save_path, f\"{time_str}_positive.txt\"))\n",
+    "        save_prompt(item.get('meta', {}).get('negativePrompt', 'No negative prompt available'), os.path.join(save_path, f\"{time_str}_negative.txt\"))\n",
+    "        save_json(item, os.path.join(save_path, f\"{time_str}.json\"))\n",
+    "    else:\n",
+    "        print(\"Item missing 'createdAt' or 'url', skipping...\")\n",
+    "\n",
+    "def download_image(url, path):\n",
+    "    try:\n",
+    "        response = requests.get(url)\n",
+    "        if response.status_code == 200:\n",
+    "            with open(path, 'wb') as f:\n",
+    "                f.write(response.content)\n",
+    "    except requests.RequestException as e:\n",
+    "        print(f\"Request failed for {url}: {e}\")\n",
+    "\n",
+    "def resize_image(image_path, max_size):\n",
+    "    with Image.open(image_path) as img:\n",
+    "        if img.width > max_size or img.height > max_size:\n",
+    "            img.thumbnail((max_size, max_size))\n",
+    "            img.save(image_path)\n",
+    "\n",
+    "def save_prompt(prompt, path):\n",
+    "    with open(path, 'w') as f:\n",
+    "        f.write(prompt)\n",
+    "\n",
+    "def save_json(data, path):\n",
+    "    with open(path, 'w', encoding='utf-8') as f:\n",
+    "        json.dump(data, f, indent=4)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "133e143d-1bb5-4145-8d59-323570bb6e95",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:58:51.566234Z",
+     "iopub.status.busy": "2025-02-08T21:58:51.565711Z",
+     "iopub.status.idle": "2025-02-08T21:58:51.568973Z",
+     "shell.execute_reply": "2025-02-08T21:58:51.568474Z",
+     "shell.execute_reply.started": "2025-02-08T21:58:51.566214Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "input_dir = current_dir.parent / 'data/sorted/image_metadata/'\n",
+    "images = current_dir.parent / 'data/sorted/images'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e11c4ad5-c040-42b7-a1b9-a0e21b33ce20",
+   "metadata": {},
+   "source": [
+    "uncomment this to download the images, otherwise proceed with grid creation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "88a6ca8e-51ee-4d27-ba0c-5f026d5750df",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:58:52.986899Z",
+     "iopub.status.busy": "2025-02-08T21:58:52.986456Z",
+     "iopub.status.idle": "2025-02-08T21:58:52.989508Z",
+     "shell.execute_reply": "2025-02-08T21:58:52.988892Z",
+     "shell.execute_reply.started": "2025-02-08T21:58:52.986878Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "#download_and_save_data(input_dir, images)  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "f247091b-7b45-4cd4-a6ca-3d4dea414e19",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:58:54.127935Z",
+     "iopub.status.busy": "2025-02-08T21:58:54.127626Z",
+     "iopub.status.idle": "2025-02-08T21:58:54.254290Z",
+     "shell.execute_reply": "2025-02-08T21:58:54.253243Z",
+     "shell.execute_reply.started": "2025-02-08T21:58:54.127914Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Grid image saved to /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/plots/grid10x120.png\n"
+     ]
+    }
+   ],
+   "source": [
+    "directory = current_dir.parent / 'data/sorted/images/'\n",
+    "#directory = '/home/lauwag/shares/laura_wagner/Civitai_page_analysis/Civitai_dataset/dataset/chronological/full/prompts-images/2024/2024-05/2024-05-31/'\n",
+    "output = current_dir.parent / 'plots/grid10x120.png'\n",
+    "\n",
+    "cell_size = 64  # Resize to 32x32 pixels\n",
+    "grid_size = (10, 120)\n",
+    "\n",
+    "\n",
+    "def apply_pixelation(img, level):\n",
+    "    \"\"\" Apply pixelation based on the level. \"\"\"\n",
+    "    if level > 1:\n",
+    "        pixel_size = 8  # Adjust this value for more or less pixelation\n",
+    "        img_small = img.resize(\n",
+    "            (img.width // pixel_size, img.height // pixel_size), Image.NEAREST\n",
+    "        )\n",
+    "        img_pixelated = img_small.resize(img.size, Image.NEAREST)\n",
+    "        return img_pixelated\n",
+    "    return img\n",
+    "\n",
+    "def apply_blur(img, level):\n",
+    "    \"\"\" Apply Gaussian blur based on the level of browsing. \"\"\"\n",
+    "    if level > 1:\n",
+    "        return img.filter(ImageFilter.GaussianBlur(radius=1.75))  # Adjust radius as needed\n",
+    "    return img\n",
+    "\n",
+    "def apply_border_and_overlay(img, color):\n",
+    "    \"\"\" Apply a colored border and a slight color overlay. \"\"\"\n",
+    "    # Convert Matplotlib color name to RGB\n",
+    "    rgb_color = tuple(int(x * 255) for x in mcolors.to_rgb(color))\n",
+    "    # Create a border\n",
+    "    border_size = 0  # Adjust border size as needed\n",
+    "    border_img = Image.new('RGB', (img.width + 2 * border_size, img.height + 2 * border_size), rgb_color)\n",
+    "    border_img.paste(img, (border_size, border_size))\n",
+    "    # Create an overlay\n",
+    "    overlay = Image.new('RGBA', border_img.size, (*rgb_color, 0))  # Semi-transparent overlay\n",
+    "    #overlay = Image.new('RGBA', border_img.size, (*rgb_color, 128))  # Semi-transparent overlay\n",
+    "    final_img = Image.alpha_composite(border_img.convert('RGBA'), overlay)\n",
+    "    return final_img.convert('RGB')\n",
+    "\n",
+    "def process_image(image_path, json_path, cell_size):\n",
+    "    \"\"\" Process each image: resize, crop, blur, and add border and overlay based on JSON metadata. \"\"\"\n",
+    "    with open(json_path, 'r') as f:\n",
+    "        metadata = json.load(f)\n",
+    "    browsing_level = metadata.get('browsingLevel', 1)\n",
+    "\n",
+    "    with Image.open(image_path) as img:\n",
+    "        img = resize_and_crop_image(img, cell_size)\n",
+    "        img = apply_pixelation(img, browsing_level)\n",
+    "\n",
+    "        color_map = {\n",
+    "            2: 'rosybrown',  # Matplotlib color name\n",
+    "            4: 'coral',\n",
+    "            8: 'red',\n",
+    "            16: 'magenta'\n",
+    "        }\n",
+    "        if browsing_level in color_map:\n",
+    "            img = apply_border_and_overlay(img, color_map[browsing_level])\n",
+    "\n",
+    "        return img\n",
+    "\n",
+    "def resize_and_crop_image(img, output_size):\n",
+    "    \"\"\" Resize and crop the image to a square of the specified size. \"\"\"\n",
+    "    ratio = min(img.width / output_size, img.height / output_size)\n",
+    "    new_size = (int(img.width / ratio), int(img.height / ratio))\n",
+    "    img = img.resize(new_size, Image.Resampling.LANCZOS)\n",
+    "    left = (img.width - output_size) // 2\n",
+    "    top = (img.height - output_size) // 2\n",
+    "    right = left + output_size\n",
+    "    bottom = top + output_size\n",
+    "    return img.crop((left, top, right, bottom))\n",
+    "\n",
+    "# Example usage and the rest of your script remains unchanged\n",
+    "\n",
+    "# Example usage\n",
+    "  # 4x4 grid\n",
+    "\n",
+    "file_types = ('png', 'jpg', 'jpeg')  # Define acceptable image file types\n",
+    "\n",
+    "images = []\n",
+    "for root, dirs, files in os.walk(directory):\n",
+    "    for file in files:\n",
+    "        if file.lower().endswith(file_types):\n",
+    "            image_path = os.path.join(root, file)\n",
+    "            json_path = image_path.rsplit('.', 1)[0] + '.json'\n",
+    "            if os.path.exists(json_path):\n",
+    "                img = process_image(image_path, json_path, cell_size)\n",
+    "                images.append(img)\n",
+    "            if len(images) == grid_size[0] * grid_size[1]:\n",
+    "                break\n",
+    "    if len(images) == grid_size[0] * grid_size[1]:\n",
+    "        break\n",
+    "\n",
+    "# Create the grid image\n",
+    "grid_img = Image.new('RGB', (grid_size[1] * cell_size, grid_size[0] * cell_size))\n",
+    "for index, img in enumerate(images):\n",
+    "    x = (index % grid_size[1]) * cell_size\n",
+    "    y = (index // grid_size[1]) * cell_size\n",
+    "    grid_img.paste(img, (x, y))\n",
+    "\n",
+    "grid_img.save(output)\n",
+    "print(f\"Grid image saved to {output}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "28bba299-c5ef-449b-a7fa-1afdc5e26262",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

scripts/Section_6-2_Demographics.ipynb

@@ -0,0 +1,1779 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d8161f06-4fd2-436d-9e59-3f68b5a67f2c",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T18:30:16.974712Z",
+     "iopub.status.busy": "2025-02-06T18:30:16.974296Z",
+     "iopub.status.idle": "2025-02-06T18:30:16.976909Z",
+     "shell.execute_reply": "2025-02-06T18:30:16.976526Z",
+     "shell.execute_reply.started": "2025-02-06T18:30:16.974692Z"
+    }
+   },
+   "source": [
+    "# Section 6.2:  Age and Gender Estimation using MiVOLO"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "64aaedec-ef56-4a62-b61e-12de2675a1ae",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T19:52:51.171282Z",
+     "iopub.status.busy": "2025-02-06T19:52:51.170711Z",
+     "iopub.status.idle": "2025-02-06T19:52:55.405039Z",
+     "shell.execute_reply": "2025-02-06T19:52:55.404308Z",
+     "shell.execute_reply.started": "2025-02-06T19:52:51.171245Z"
+    }
+   },
+   "source": [
+    "![Alt text](../plots/mivolo.svg)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "4293f307-44fd-455e-90fe-6e6928be9af5",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:59:21.970807Z",
+     "iopub.status.busy": "2025-02-08T21:59:21.969931Z",
+     "iopub.status.idle": "2025-02-08T22:00:09.724295Z",
+     "shell.execute_reply": "2025-02-08T22:00:09.723583Z",
+     "shell.execute_reply.started": "2025-02-08T21:59:21.970784Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/lauwag/data/conda/envs/horde/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import csv\n",
+    "from pathlib import Path\n",
+    "import logging\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import requests\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "import cv2\n",
+    "from io import BytesIO\n",
+    "from PIL import Image, UnidentifiedImageError\n",
+    "from datetime import datetime, timedelta\n",
+    "from dateutil.relativedelta import relativedelta\n",
+    "from mivolo.predictor import Predictor\n",
+    "import matplotlib.pyplot as plt\n",
+    "import matplotlib.patches as mpatches"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "63a54f5f-900c-48dd-8932-2632e56c5670",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:09.726069Z",
+     "iopub.status.busy": "2025-02-08T22:00:09.725699Z",
+     "iopub.status.idle": "2025-02-08T22:00:09.730626Z",
+     "shell.execute_reply": "2025-02-08T22:00:09.730099Z",
+     "shell.execute_reply.started": "2025-02-08T22:00:09.726050Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "current_dir = Path.cwd()\n",
+    "mini = current_dir.parent / 'data/CSV/image_subsets/Civiverse-mini.csv'\n",
+    "mivolo_in = current_dir.parent / 'data/CSV/image_subsets/Civiverse-mini-by-month/'\n",
+    "(current_dir.parent /  'data/CSV/image_subsets/Civiverse-mini-by-month/').mkdir(parents=True, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1fdfb89a-6094-4382-8755-fae213221ea5",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:09.731540Z",
+     "iopub.status.busy": "2025-02-08T22:00:09.731359Z",
+     "iopub.status.idle": "2025-02-08T22:00:09.825738Z",
+     "shell.execute_reply": "2025-02-08T22:00:09.825258Z",
+     "shell.execute_reply.started": "2025-02-08T22:00:09.731524Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def split_by_month(input_path, output_dir):\n",
+    "    # Load the dataset\n",
+    "    df = pd.read_csv(input_path)\n",
+    "    \n",
+    "    # Convert the 'createdAt' column to datetime\n",
+    "    df['createdAt'] = pd.to_datetime(df['createdAt'], errors='coerce')\n",
+    "    \n",
+    "    # Extract year and month\n",
+    "    df['year_month'] = df['createdAt'].dt.to_period('M')\n",
+    "    \n",
+    "    # Group the data by year and month and save each group as a CSV file\n",
+    "    unique_months = df['year_month'].unique()\n",
+    "\n",
+    "    for month in unique_months:\n",
+    "        # Filter data for the specific month\n",
+    "        df_month = df[df['year_month'] == month]\n",
+    "        \n",
+    "        # Define the file name based on the year and month\n",
+    "        file_name = f'{output_dir}/Civiverse-{month}.csv'\n",
+    "        \n",
+    "        # Save the file\n",
+    "        df_month.to_csv(file_name, index=False)\n",
+    "\n",
+    "    print(f\"Data has been split and saved to {output_dir}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "2c909d7c-7d16-4dc7-8364-7f1c0784414c",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:09.827095Z",
+     "iopub.status.busy": "2025-02-08T22:00:09.826919Z",
+     "iopub.status.idle": "2025-02-08T22:00:10.479484Z",
+     "shell.execute_reply": "2025-02-08T22:00:10.478777Z",
+     "shell.execute_reply.started": "2025-02-08T22:00:09.827079Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/sctmp/lauwag/ipykernel_1497673/1825509207.py:9: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
+      "  df['year_month'] = df['createdAt'].dt.to_period('M')\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Data has been split and saved to /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month\n"
+     ]
+    }
+   ],
+   "source": [
+    "split_by_month(mini, mivolo_in)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "4c543306-ffc8-4b9c-a3df-b49b2271caa9",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:10.480505Z",
+     "iopub.status.busy": "2025-02-08T22:00:10.480310Z",
+     "iopub.status.idle": "2025-02-08T22:00:10.483961Z",
+     "shell.execute_reply": "2025-02-08T22:00:10.483400Z",
+     "shell.execute_reply.started": "2025-02-08T22:00:10.480486Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "mivolo_out = current_dir.parent / 'data/CSV/image_subsets/MiVOLO-results/'\n",
+    "mivolo_out.mkdir(parents=True, exist_ok=True)  # Create the output directory if it doesn't exist"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "304ed12f-c7b6-4129-b24d-7ccc793a62c7",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:10.484802Z",
+     "iopub.status.busy": "2025-02-08T22:00:10.484639Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/lauwag/data/conda/envs/horde/lib/python3.12/site-packages/ultralytics/nn/tasks.py:634: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  return torch.load(file, map_location=\"cpu\"), file  # load\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model summary (fused): 268 layers, 68125494 parameters, 0 gradients, 257.4 GFLOPs\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[W208 23:00:15.738708520 NNPACK.cpp:61] Could not initialize NNPACK! Reason: Unsupported hardware.\n",
+      "/home/lauwag/data/conda/envs/horde/lib/python3.12/site-packages/mivolo/model/mi_volo.py:33: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  state = torch.load(ckpt_path, map_location=\"cpu\")\n",
+      "INFO:MiVOLO:Model meta:\n",
+      "min_age: 1, max_age: 95, avg_age: 48.0, num_classes: 3, in_chans: 6, with_persons_model: True, disable_faces: False, use_persons: True, only_age: False, num_classes_gender: 2, input_size: 224, use_person_crops: True, use_face_crops: True\n",
+      "/home/lauwag/data/conda/envs/horde/lib/python3.12/site-packages/timm/models/_helpers.py:39: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  checkpoint = torch.load(checkpoint_path, map_location='cpu')\n",
+      "INFO:timm.models._helpers:Loaded state_dict from checkpoint '/shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/ext/MiVOLO/models/mivolo_imbd.pth.tar'\n",
+      "INFO:MiVOLO:Model mivolo_d1_224 created, param count: 27432414\n",
+      "INFO:timm.data.config:Data processing configuration for current model + dataset:\n",
+      "INFO:timm.data.config:\tinput_size: (3, 224, 224)\n",
+      "INFO:timm.data.config:\tinterpolation: bicubic\n",
+      "INFO:timm.data.config:\tmean: (0.485, 0.456, 0.406)\n",
+      "INFO:timm.data.config:\tstd: (0.229, 0.224, 0.225)\n",
+      "INFO:timm.data.config:\tcrop_pct: 0.96\n",
+      "INFO:timm.data.config:\tcrop_mode: center\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2022-11.csv\n",
+      "\n",
+      "0: 640x640 (no detections), 723.9ms\n",
+      "Speed: 12.1ms preprocess, 723.9ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "Processed and saved 1 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2022-11.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2022-12.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-01.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-02.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-03.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-04.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-05.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-06.csv\n",
+      "\n",
+      "0: 416x640 1 person, 455.1ms\n",
+      "Speed: 3.5ms preprocess, 455.1ms inference, 33.5ms postprocess per image at shape (1, 3, 416, 640)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 32.89\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-06.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-07.csv\n",
+      "\n",
+      "0: 640x320 1 person, 395.7ms\n",
+      "Speed: 2.9ms preprocess, 395.7ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 320)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 33.49\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 478.5ms\n",
+      "Speed: 1.9ms preprocess, 478.5ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.81\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 2 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-07.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-08.csv\n",
+      "\n",
+      "0: 640x448 1 person, 478.0ms\n",
+      "Speed: 2.9ms preprocess, 478.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 40.62\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-08.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-09.csv\n",
+      "\n",
+      "0: 416x640 (no detections), 567.1ms\n",
+      "Speed: 2.4ms preprocess, 567.1ms inference, 0.4ms postprocess per image at shape (1, 3, 416, 640)\n",
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 320x640 (no detections), 393.6ms\n",
+      "Speed: 1.7ms preprocess, 393.6ms inference, 0.4ms postprocess per image at shape (1, 3, 320, 640)\n",
+      "\n",
+      "0: 640x640 (no detections), 711.9ms\n",
+      "Speed: 3.4ms preprocess, 711.9ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x640 (no detections), 699.8ms\n",
+      "Speed: 2.3ms preprocess, 699.8ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x576 1 person, 629.6ms\n",
+      "Speed: 2.4ms preprocess, 629.6ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 576)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 28.65\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 598.3ms\n",
+      "Speed: 2.1ms preprocess, 598.3ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 25.85\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/98848c97-3d1e-4b52-9967-aeeca354a30e/width=656/98848c97-3d1e-4b52-9967-aeeca354a30e.mp4: cannot identify image file <_io.BytesIO object at 0x14cb00133740>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-09.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-10.csv\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/e6469288-b487-4a06-99c1-59e7ac22fa77/width=1024/e6469288-b487-4a06-99c1-59e7ac22fa77.mp4: cannot identify image file <_io.BytesIO object at 0x14cb00ecbdd0>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 448x640 (no detections), 536.6ms\n",
+      "Speed: 10.1ms preprocess, 536.6ms inference, 0.4ms postprocess per image at shape (1, 3, 448, 640)\n",
+      "Processed and saved 2 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-10.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-11.csv\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 662.9ms\n",
+      "Speed: 2.6ms preprocess, 662.9ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.0\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x384 1 person, 895.9ms\n",
+      "Speed: 2.0ms preprocess, 895.9ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 384)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 43.33\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 529.4ms\n",
+      "Speed: 2.6ms preprocess, 529.4ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 539.3ms\n",
+      "Speed: 2.8ms preprocess, 539.3ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 39.15\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 708.6ms\n",
+      "Speed: 2.5ms preprocess, 708.6ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 29.64\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/ad4b5481-0a58-4196-a3e8-fca2fe22a3cc/width=1080/ad4b5481-0a58-4196-a3e8-fca2fe22a3cc.mp4: cannot identify image file <_io.BytesIO object at 0x14cb010c24d0>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-11.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-12.csv\n",
+      "\n",
+      "0: 640x384 1 person, 1 face, 461.0ms\n",
+      "Speed: 2.4ms preprocess, 461.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 384)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 19.61\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 501.3ms\n",
+      "Speed: 3.1ms preprocess, 501.3ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.58\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/3004b5fa-af81-4de7-829d-1d809d70b878/width=512/3004b5fa-af81-4de7-829d-1d809d70b878.mp4: cannot identify image file <_io.BytesIO object at 0x14cb0020ad40>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 (no detections), 842.5ms\n",
+      "Speed: 4.5ms preprocess, 842.5ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x416 (no detections), 446.8ms\n",
+      "Speed: 2.5ms preprocess, 446.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 416)\n",
+      "Processed and saved 5 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-12.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-01.csv\n",
+      "\n",
+      "0: 640x448 (no detections), 638.5ms\n",
+      "Speed: 2.3ms preprocess, 638.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x416 (no detections), 441.7ms\n",
+      "Speed: 2.5ms preprocess, 441.7ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 416)\n",
+      "\n",
+      "0: 640x448 (no detections), 470.3ms\n",
+      "Speed: 2.3ms preprocess, 470.3ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 693.9ms\n",
+      "Speed: 2.5ms preprocess, 693.9ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x512 1 person, 1 face, 808.6ms\n",
+      "Speed: 3.2ms preprocess, 808.6ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 15.01\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x320 1 person, 1 face, 345.6ms\n",
+      "Speed: 2.0ms preprocess, 345.6ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 320)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.86\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 6 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-01.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-02.csv\n",
+      "\n",
+      "0: 640x384 1 person, 1 face, 387.8ms\n",
+      "Speed: 1.9ms preprocess, 387.8ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 384)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.31\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x480 1 person, 1 face, 540.4ms\n",
+      "Speed: 2.5ms preprocess, 540.4ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 480)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.47\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 1 person, 1 face, 713.1ms\n",
+      "Speed: 3.8ms preprocess, 713.1ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 640)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.85\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 (no detections), 778.8ms\n",
+      "Speed: 28.7ms preprocess, 778.8ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 528.2ms\n",
+      "Speed: 2.3ms preprocess, 528.2ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.63\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 518.4ms\n",
+      "Speed: 3.9ms preprocess, 518.4ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 18.25\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 470.7ms\n",
+      "Speed: 2.5ms preprocess, 470.7ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.51\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x480 1 person, 1 face, 647.1ms\n",
+      "Speed: 2.4ms preprocess, 647.1ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 480)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 58.87\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 469.8ms\n",
+      "Speed: 2.6ms preprocess, 469.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 477.5ms\n",
+      "Speed: 2.3ms preprocess, 477.5ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 23.79\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 10 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-02.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-03.csv\n",
+      "\n",
+      "0: 640x448 1 face, 511.4ms\n",
+      "Speed: 2.5ms preprocess, 511.4ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 24.87\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x544 (no detections), 576.5ms\n",
+      "Speed: 2.9ms preprocess, 576.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 544)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 687.1ms\n",
+      "Speed: 9.9ms preprocess, 687.1ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 25.76\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 498.3ms\n",
+      "Speed: 2.3ms preprocess, 498.3ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x512 (no detections), 573.2ms\n",
+      "Speed: 3.0ms preprocess, 573.2ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 512)\n",
+      "Processed and saved 5 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-03.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-04.csv\n",
+      "\n",
+      "0: 640x384 (no detections), 518.2ms\n",
+      "Speed: 2.7ms preprocess, 518.2ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 384)\n",
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x512 (no detections), 707.7ms\n",
+      "Speed: 3.6ms preprocess, 707.7ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 512)\n",
+      "\n",
+      "0: 640x416 (no detections), 453.7ms\n",
+      "Speed: 2.4ms preprocess, 453.7ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 416)\n",
+      "\n",
+      "0: 640x384 (no detections), 391.0ms\n",
+      "Speed: 2.0ms preprocess, 391.0ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 384)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 449.8ms\n",
+      "Speed: 2.3ms preprocess, 449.8ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.39\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 618.4ms\n",
+      "Speed: 2.3ms preprocess, 618.4ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 631.0ms\n",
+      "Speed: 2.2ms preprocess, 631.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 24.05\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x512 1 person, 496.4ms\n",
+      "Speed: 2.6ms preprocess, 496.4ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.81\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 442.8ms\n",
+      "Speed: 2.3ms preprocess, 442.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 477.7ms\n",
+      "Speed: 2.4ms preprocess, 477.7ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.62\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 447.0ms\n",
+      "Speed: 2.2ms preprocess, 447.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 54.31\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 (no detections), 819.0ms\n",
+      "Speed: 3.6ms preprocess, 819.0ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 478.2ms\n",
+      "Speed: 1.8ms preprocess, 478.2ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.56\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 471.2ms\n",
+      "Speed: 2.7ms preprocess, 471.2ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.31\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 484.0ms\n",
+      "Speed: 2.2ms preprocess, 484.0ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x640 (no detections), 832.6ms\n",
+      "Speed: 3.0ms preprocess, 832.6ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 508.9ms\n",
+      "Speed: 2.5ms preprocess, 508.9ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 27.19\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6879c7b9-5cb3-42db-b409-30b4e2f71945/width=1080/6879c7b9-5cb3-42db-b409-30b4e2f71945.mp4: cannot identify image file <_io.BytesIO object at 0x14cb0020ad40>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 9 persons, 461.8ms\n",
+      "Speed: 2.4ms preprocess, 461.8ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([9, 3, 224, 224]), person_input: torch.Size([9, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 30.4\n",
+      "INFO:MiVOLO:\tgender: male [55%]\n",
+      "INFO:MiVOLO:\tage: 28.89\n",
+      "INFO:MiVOLO:\tgender: female [63%]\n",
+      "INFO:MiVOLO:\tage: 30.31\n",
+      "INFO:MiVOLO:\tgender: female [68%]\n",
+      "INFO:MiVOLO:\tage: 31.62\n",
+      "INFO:MiVOLO:\tgender: female [53%]\n",
+      "INFO:MiVOLO:\tage: 35.17\n",
+      "INFO:MiVOLO:\tgender: male [53%]\n",
+      "INFO:MiVOLO:\tage: 33.02\n",
+      "INFO:MiVOLO:\tgender: male [95%]\n",
+      "INFO:MiVOLO:\tage: 35.17\n",
+      "INFO:MiVOLO:\tgender: male [53%]\n",
+      "INFO:MiVOLO:\tage: 35.17\n",
+      "INFO:MiVOLO:\tgender: male [53%]\n",
+      "INFO:MiVOLO:\tage: 35.17\n",
+      "INFO:MiVOLO:\tgender: male [53%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 19 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-04.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-05.csv\n",
+      "\n",
+      "0: 640x448 1 person, 455.5ms\n",
+      "Speed: 2.2ms preprocess, 455.5ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 37.57\n",
+      "INFO:MiVOLO:\tgender: male [95%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 438.7ms\n",
+      "Speed: 2.2ms preprocess, 438.7ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 15.62\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 444.8ms\n",
+      "Speed: 2.3ms preprocess, 444.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6032bd70-6d53-4007-9e89-e69d4748efb5/width=528/6032bd70-6d53-4007-9e89-e69d4748efb5.mp4: cannot identify image file <_io.BytesIO object at 0x14cb00ed0950>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 453.9ms\n",
+      "Speed: 2.3ms preprocess, 453.9ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 475.0ms\n",
+      "Speed: 1.6ms preprocess, 475.0ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.5\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 447.6ms\n",
+      "Speed: 2.5ms preprocess, 447.6ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 23.46\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x512 (no detections), 528.5ms\n",
+      "Speed: 3.2ms preprocess, 528.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 512)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 449.8ms\n",
+      "Speed: 2.3ms preprocess, 449.8ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 29.32\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 617.7ms\n",
+      "Speed: 2.4ms preprocess, 617.7ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.32\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 609.1ms\n",
+      "Speed: 2.3ms preprocess, 609.1ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 436.2ms\n",
+      "Speed: 2.5ms preprocess, 436.2ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x512 1 person, 1 face, 585.6ms\n",
+      "Speed: 3.1ms preprocess, 585.6ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.5\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 457.3ms\n",
+      "Speed: 2.1ms preprocess, 457.3ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 25.19\n",
+      "INFO:MiVOLO:\tgender: male [81%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 14 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-05.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-06.csv\n",
+      "\n",
+      "0: 640x448 (no detections), 484.5ms\n",
+      "Speed: 2.8ms preprocess, 484.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x512 (no detections), 524.8ms\n",
+      "Speed: 2.9ms preprocess, 524.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 512)\n",
+      "\n",
+      "0: 640x480 1 person, 478.0ms\n",
+      "Speed: 2.6ms preprocess, 478.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 480)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 39.4\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x512 1 person, 1 face, 539.8ms\n",
+      "Speed: 2.6ms preprocess, 539.8ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.33\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 2 faces, 446.7ms\n",
+      "Speed: 2.4ms preprocess, 446.7ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([2, 3, 224, 224]), person_input: torch.Size([2, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.65\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "INFO:MiVOLO:\tage: 20.53\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 1 person, 1 face, 655.0ms\n",
+      "Speed: 3.3ms preprocess, 655.0ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 640)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 26.34\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x384 (no detections), 400.6ms\n",
+      "Speed: 2.1ms preprocess, 400.6ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 384)\n",
+      "\n",
+      "0: 640x448 1 person, 587.9ms\n",
+      "Speed: 2.2ms preprocess, 587.9ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 30.4\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 610.3ms\n",
+      "Speed: 2.3ms preprocess, 610.3ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 453.6ms\n",
+      "Speed: 2.3ms preprocess, 453.6ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x512 1 person, 1 face, 511.3ms\n",
+      "Speed: 2.8ms preprocess, 511.3ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 34.28\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 441.2ms\n",
+      "Speed: 2.3ms preprocess, 441.2ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 586.3ms\n",
+      "Speed: 2.3ms preprocess, 586.3ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 437.5ms\n",
+      "Speed: 2.4ms preprocess, 437.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 448x640 1 person, 1 face, 437.4ms\n",
+      "Speed: 2.4ms preprocess, 437.4ms inference, 0.7ms postprocess per image at shape (1, 3, 448, 640)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.81\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 436.8ms\n",
+      "Speed: 2.6ms preprocess, 436.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 448x640 (no detections), 433.0ms\n",
+      "Speed: 1.9ms preprocess, 433.0ms inference, 0.4ms postprocess per image at shape (1, 3, 448, 640)\n",
+      "\n",
+      "0: 640x448 (no detections), 599.7ms\n",
+      "Speed: 2.5ms preprocess, 599.7ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Set up logging\n",
+    "detector_weights = current_dir.parent / 'ext/MiVOLO/models/yolov8x_person_face.pt'\n",
+    "checkpoint = current_dir.parent / 'ext/MiVOLO/models/mivolo_imbd.pth.tar'\n",
+    "\n",
+    "_logger = logging.getLogger(\"inference\")\n",
+    "logging.basicConfig(level=logging.INFO)\n",
+    "\n",
+    "# Placeholder configuration and predictor initialization for MiVOLO\n",
+    "class Config:\n",
+    "    def __init__(self, detector_weights, checkpoint, device, with_persons=True, disable_faces=False, draw=False):\n",
+    "        self.detector_weights = detector_weights\n",
+    "        self.checkpoint = checkpoint\n",
+    "        self.device = device\n",
+    "        self.with_persons = with_persons\n",
+    "        self.disable_faces = disable_faces\n",
+    "        self.draw = draw\n",
+    "\n",
+    "\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "config = Config(detector_weights=detector_weights, checkpoint=checkpoint, device=device)\n",
+    "predictor = Predictor(config, verbose=True)\n",
+    "\n",
+    "def download_image(url):\n",
+    "    try:\n",
+    "        response = requests.get(url)\n",
+    "        response.raise_for_status()\n",
+    "        return Image.open(BytesIO(response.content)).convert(\"RGB\")\n",
+    "    except requests.RequestException as e:\n",
+    "        _logger.error(f\"Failed to download image from {url}: {e}\")\n",
+    "        return None\n",
+    "    except UnidentifiedImageError as e:\n",
+    "        _logger.error(f\"Unidentified image error for URL {url}: {e}\")\n",
+    "        return None\n",
+    "\n",
+    "def process_images_with_progress(data, predictor, output_file, start_idx=0):\n",
+    "    results = []\n",
+    "    total_images = len(data)\n",
+    "\n",
+    "    for idx, row in data.iterrows():\n",
+    "        if idx < start_idx:\n",
+    "            continue\n",
+    "\n",
+    "        img_url = row[\"url\"]\n",
+    "        pil_image = download_image(img_url)\n",
+    "        if pil_image is None:\n",
+    "            continue\n",
+    "\n",
+    "        np_image = np.array(pil_image)\n",
+    "        np_image = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR)\n",
+    "        detected_objects, _ = predictor.recognize(np_image)\n",
+    "\n",
+    "        row_result = row.to_dict()  # Start with the original row's data\n",
+    "\n",
+    "        if detected_objects and detected_objects.ages:\n",
+    "            for i in range(len(detected_objects.ages)):\n",
+    "                age = detected_objects.ages[i]\n",
+    "                gender = detected_objects.genders[i]\n",
+    "                gender_confidence = detected_objects.gender_scores[i]\n",
+    "\n",
+    "                if gender_confidence >= 0.83:\n",
+    "                    detection = {\n",
+    "                        \"detection_type\": 'face' if i in detected_objects.face_to_person_map else 'person',\n",
+    "                        \"gender\": gender,\n",
+    "                        \"gender_confidence\": gender_confidence,\n",
+    "                        \"age\": age,\n",
+    "                        \"n_persons\": detected_objects.n_persons,\n",
+    "                        \"n_faces\": detected_objects.n_faces,\n",
+    "                        \"detected\": True\n",
+    "                    }\n",
+    "                else:\n",
+    "                    detection = {\n",
+    "                        \"detection_type\": \"N/A\",\n",
+    "                        \"gender\": \"N/A\",\n",
+    "                        \"gender_confidence\": 0,\n",
+    "                        \"age\": 0,\n",
+    "                        \"n_persons\": detected_objects.n_persons if detected_objects else 0,\n",
+    "                        \"n_faces\": detected_objects.n_faces if detected_objects else 0,\n",
+    "                        \"detected\": False\n",
+    "                    }\n",
+    "\n",
+    "                results.append({**row_result, **detection})\n",
+    "        else:\n",
+    "            detection = {\n",
+    "                \"detection_type\": \"N/A\",\n",
+    "                \"gender\": \"N/A\",\n",
+    "                \"gender_confidence\": 0,\n",
+    "                \"age\": 0,\n",
+    "                \"n_persons\": detected_objects.n_persons if detected_objects else 0,\n",
+    "                \"n_faces\": detected_objects.n_faces if detected_objects else 0,\n",
+    "                \"detected\": False\n",
+    "            }\n",
+    "            results.append({**row_result, **detection})\n",
+    "\n",
+    "        if idx % 100 == 0 or idx == total_images - 1:\n",
+    "            df = pd.DataFrame(results)\n",
+    "            if os.path.exists(output_file):\n",
+    "                df.to_csv(output_file, mode='a', header=False, index=False)\n",
+    "            else:\n",
+    "                df.to_csv(output_file, mode='w', header=True, index=False)\n",
+    "            results = []\n",
+    "            print(f\"Processed and saved {idx + 1} images so far.\")\n",
+    "\n",
+    "def generate_months(start, end):\n",
+    "    start_date = datetime.strptime(start, '%Y-%m')\n",
+    "    end_date = datetime.strptime(end, '%Y-%m')\n",
+    "    while start_date <= end_date:\n",
+    "        yield start_date.strftime('%Y-%m')\n",
+    "        start_date += relativedelta(months=1)  # Increment by calendar months\n",
+    "\n",
+    "\n",
+    "start_month = '2022-11'\n",
+    "end_month = '2024-12'\n",
+    "\n",
+    "for month in generate_months(start_month, end_month):\n",
+    "    input_file_path = mivolo_in / f'Civiverse-{month}.csv'\n",
+    "    output_file_path = mivolo_out / f'{month}.csv'\n",
+    "\n",
+    "    if input_file_path.exists():\n",
+    "        print(f\"Processing: {input_file_path}\")\n",
+    "\n",
+    "        data = pd.read_csv(input_file_path)\n",
+    "        start_index = 0\n",
+    "        process_images_with_progress(data, predictor, output_file_path, start_idx=start_index)\n",
+    "\n",
+    "        print(f\"Processed and saved to: {output_file_path}\")\n",
+    "    else:\n",
+    "        print(f\"File not found: {input_file_path}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "88ec896a-bf9b-4cc6-a787-c1343f8acb41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import matplotlib.patches as mpatches\n",
+    "\n",
+    "input_dir = current_dir.parent / 'data/CSV/image_subsets/MiVOLO-results/'\n",
+    "plot_dir = current_dir.parent / 'plots/'\n",
+    "\n",
+    "\n",
+    "all_data = pd.DataFrame()\n",
+    "for file_path in input_dir.glob('*.csv'):  # Reads all CSV files in the folder\n",
+    "    #print(f\"Loading: {file_path}\")\n",
+    "    data = pd.read_csv(file_path)\n",
+    "    all_data = pd.concat([all_data, data], ignore_index=True)\n",
+    "\n",
+    "# Filter rows where detection_type equals \"person\"\n",
+    "person_data = all_data[all_data['detection_type'] == 'person']\n",
+    "\n",
+    "# Count unique images and categorize by persons detected\n",
+    "n_images = all_data['id'].nunique()\n",
+    "images_with_zero_persons = all_data[all_data['n_persons'] == 0]['id'].nunique()\n",
+    "images_with_one_or_more_persons = all_data[all_data['n_persons'] > 0]['id'].nunique()\n",
+    "\n",
+    "n_persons_detected = person_data['id'].nunique()  # Unique images with at least one detected person\n",
+    "total_persons_detected = person_data.shape[0]  # Total number of persons detected\n",
+    "\n",
+    "\n",
+    "\n",
+    "n_total_female = person_data[person_data['gender'] == 'female']['id'].nunique()\n",
+    "n_total_male = person_data[person_data['gender'] == 'male']['id'].nunique()\n",
+    "\n",
+    "# Filter the data further for non-missing age and gender\n",
+    "filtered_data = person_data.dropna(subset=['age', 'gender'])\n",
+    "\n",
+    "# Round ages for consistent plotting\n",
+    "filtered_data['rounded_age'] = np.round(filtered_data['age'] * 4) / 4\n",
+    "\n",
+    "# Map browsingLevel to colors\n",
+    "def get_browsing_color(browsing_level):\n",
+    "    color_mapping = {\n",
+    "        1: 'silver',\n",
+    "        2: 'rosybrown',\n",
+    "        4: 'coral',\n",
+    "        8: 'crimson',\n",
+    "        16: 'blueviolet'\n",
+    "    }\n",
+    "    return color_mapping.get(browsing_level, 'black')  # Default to black for unknown values\n",
+    "\n",
+    "filtered_data['color'] = filtered_data['browsingLevel'].apply(get_browsing_color)\n",
+    "\n",
+    "# Aggregate data for plotting\n",
+    "aggregated_data = (\n",
+    "    filtered_data.groupby(['rounded_age', 'gender', 'color'])\n",
+    "    .size()\n",
+    "    .unstack(fill_value=0)\n",
+    ")\n",
+    "\n",
+    "# Define NSFW colors\n",
+    "nsfw_colors = ['blueviolet', 'crimson', 'coral', 'rosybrown', 'silver']\n",
+    "\n",
+    "# Plotting function\n",
+    "def plot_gender_data(ax, data, gender_label):\n",
+    "    ages = data.index\n",
+    "    bottom = np.zeros(len(ages))\n",
+    "    \n",
+    "    for color in nsfw_colors:\n",
+    "        counts = data[color] if color in data.columns else np.zeros(len(ages))\n",
+    "        ax.bar(\n",
+    "            ages,\n",
+    "            counts,\n",
+    "            color=color,\n",
+    "            edgecolor=color,\n",
+    "            linewidth=1,\n",
+    "            width=0.2,\n",
+    "            bottom=bottom,\n",
+    "            alpha=0.5\n",
+    "        )\n",
+    "        bottom += counts\n",
+    "\n",
+    "    x_min = 5\n",
+    "    x_max = filtered_data['rounded_age'].max()\n",
+    "    ax.set_xticks(np.arange(x_min, x_max + 0.5, 5))\n",
+    "    ax.set_xticklabels([f'{int(age)}' for age in np.arange(x_min, x_max + 0.5, 5)], fontsize=12, fontweight='bold')\n",
+    "    ax.set_xticks(np.arange(x_min, x_max + 0.5, 0.5), minor=True)\n",
+    "\n",
+    "    y_min = 0\n",
+    "    y_max = bottom.max() + 100\n",
+    "    y_ticks = np.arange(y_min, y_max + 1, 100)  # Fine-grained steps of 100\n",
+    "    ax.set_yticks(y_ticks)\n",
+    "    ax.set_yticklabels([str(int(y)) for y in y_ticks], fontsize=12, fontweight='bold')\n",
+    "\n",
+    "    ax.grid(True, which='major', color='lightgrey', linestyle='-', linewidth=0.5)\n",
+    "    ax.grid(True, which='minor', color='lightgrey', linestyle=':', linewidth=0.5)\n",
+    "\n",
+    "    ax.spines['top'].set_visible(False)\n",
+    "    ax.spines['right'].set_visible(False)\n",
+    "    ax.spines['left'].set_visible(False)\n",
+    "    ax.spines['bottom'].set_visible(False)\n",
+    "    \n",
+    "    ax.set_xlabel('Age', fontsize=12, fontweight='bold')\n",
+    "    if gender_label == 'Female':\n",
+    "        ax.set_ylabel('Number of Subjects', fontsize=14, fontweight='bold')\n",
+    "    ax.set_title(f'{gender_label} Read', fontsize=14, fontweight='bold')\n",
+    "\n",
+    "# Set up the subplots\n",
+    "fig, axes = plt.subplots(1, 2, figsize=(14, 6.5), sharey=True)\n",
+    "\n",
+    "plot_gender_data(axes[0], aggregated_data.xs('male', level='gender'), 'Male')\n",
+    "plot_gender_data(axes[1], aggregated_data.xs('female', level='gender'), 'Female')\n",
+    "\n",
+    "legend_patches = [\n",
+    "    mpatches.Patch(facecolor='blueviolet', edgecolor='blueviolet', linewidth=2, label='Level 16: XXX'),\n",
+    "    mpatches.Patch(facecolor='crimson', edgecolor='crimson', linewidth=2, label='Level 8: X'),\n",
+    "    mpatches.Patch(facecolor='coral', edgecolor='coral', linewidth=2, label='Level 4: Mature'),\n",
+    "    mpatches.Patch(facecolor='rosybrown', edgecolor='rosybrown', linewidth=2, label='Level 2: Soft'),\n",
+    "    mpatches.Patch(facecolor='silver', edgecolor='silver', linewidth=2, label='Level 1: SFW'),\n",
+    "    mpatches.Patch(facecolor='none', edgecolor='none', label=f'n images: {n_images}', alpha=0),\n",
+    "    mpatches.Patch(facecolor='none', edgecolor='none', label=f'Total persons detected: {total_persons_detected}', alpha=0),\n",
+    "    mpatches.Patch(facecolor='none', edgecolor='none', label=f'Unique images containing persons: {n_persons_detected}', alpha=0),\n",
+    "]\n",
+    "\n",
+    "axes[0].legend(handles=legend_patches, title=\"Browsing Levels\", loc='upper left', fontsize=12, title_fontsize=12, frameon=True)\n",
+    "plt.savefig(f'{plot_dir}/mivolo.svg', format='svg', bbox_inches='tight')\n",
+    "plt.tight_layout()\n",
+    "\n",
+    "# Count images with at least one person\n",
+    "images_with_one_or_more_persons = all_data[all_data['n_persons'] > 0]['id'].nunique()\n",
+    "\n",
+    "# Count unique images in `person_data`\n",
+    "n_persons_detected = person_data['id'].nunique()\n",
+    "\n",
+    "# Count total persons detected\n",
+    "total_persons = person_data.shape[0]  # This counts all detected persons\n",
+    "\n",
+    "# Display potential inconsistencies\n",
+    "print(f\"Total images: {n_images}\")\n",
+    "print(f\"Images with at least one person: {images_with_one_or_more_persons}\")\n",
+    "print(f\"Unique images in `person_data`: {n_persons_detected}\")\n",
+    "print(f\"Total number of persons detected: {total_persons}\")\n",
+    "\n",
+    "\n",
+    "\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "42b2a557-b8f4-4d0f-8907-98e3012a1b34",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T20:01:54.848400Z",
+     "iopub.status.busy": "2025-02-06T20:01:54.847713Z",
+     "iopub.status.idle": "2025-02-06T20:01:54.851533Z",
+     "shell.execute_reply": "2025-02-06T20:01:54.851102Z",
+     "shell.execute_reply.started": "2025-02-06T20:01:54.848376Z"
+    }
+   },
+   "source": [
+    "### Latex Table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e506c41-6497-4ece-99f3-73f09fe1129e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Define the directory containing CSV files\n",
+    "directory_path = current_dir.parent / 'data/CSV/image_subsets/MiVOLO-results/'  # Update this path with the actual directory path\n",
+    "\n",
+    "# Prepare data for LaTeX table\n",
+    "table_rows = []\n",
+    "\n",
+    "# Loop through each file in the directory\n",
+    "for file_name in os.listdir(directory_path):\n",
+    "    if file_name.endswith('.csv'):\n",
+    "        file_path = os.path.join(directory_path, file_name)\n",
+    "        print(f\"Processing file: {file_name}\")\n",
+    "\n",
+    "        # Load the data\n",
+    "        data = pd.read_csv(file_path)\n",
+    "\n",
+    "        # Total images analyzed\n",
+    "        total_images = data['id'].nunique()\n",
+    "\n",
+    "        # Count of images with no persons detected\n",
+    "        images_no_persons = data[data['n_persons'] == 0]['id'].nunique()\n",
+    "\n",
+    "        # Total persons detected (only using \"person\" detection type)\n",
+    "        total_persons_count = data[data['detection_type'] == 'person'].shape[0]\n",
+    "\n",
+    "        # Average age and standard deviation for male and female individuals\n",
+    "        male_age_stats = data[data['gender'] == 'male']['age'].agg(['mean', 'std']).fillna(0)\n",
+    "        female_age_stats = data[data['gender'] == 'female']['age'].agg(['mean', 'std']).fillna(0)\n",
+    "\n",
+    "        # Count of female and male subjects\n",
+    "        female_images_count = data[data['gender'] == 'female']['id'].nunique()\n",
+    "        male_images_count = data[data['gender'] == 'male']['id'].nunique()\n",
+    "\n",
+    "        # Female to male ratio\n",
+    "        female_to_male_ratio = female_images_count / male_images_count if male_images_count else None\n",
+    "\n",
+    "        # Browsing level analysis for females\n",
+    "        female_browsing_level_1 = data[(data['gender'] == 'female') & (data['browsingLevel'] == 1)]['id'].nunique()\n",
+    "        female_browsing_level_2_16 = data[(data['gender'] == 'female') & (data['browsingLevel'].isin([2, 4, 8, 16]))]['id'].nunique()\n",
+    "        \n",
+    "        female_browsing_level_1_percentage = (female_browsing_level_1 / female_images_count * 100) if female_images_count else 0\n",
+    "        female_browsing_level_2_16_percentage = (female_browsing_level_2_16 / female_images_count * 100) if female_images_count else 0\n",
+    "\n",
+    "        # Browsing level analysis for males\n",
+    "        male_browsing_level_1 = data[(data['gender'] == 'male') & (data['browsingLevel'] == 1)]['id'].nunique()\n",
+    "        male_browsing_level_2_16 = data[(data['gender'] == 'male') & (data['browsingLevel'].isin([2, 4, 8, 16]))]['id'].nunique()\n",
+    "\n",
+    "        male_browsing_level_1_percentage = (male_browsing_level_1 / male_images_count * 100) if male_images_count else 0\n",
+    "        male_browsing_level_2_16_percentage = (male_browsing_level_2_16 / male_images_count * 100) if male_images_count else 0\n",
+    "\n",
+    "        # Add row to table data\n",
+    "        table_rows.append([\n",
+    "            file_name.replace('.csv', ''),  # Remove file extension\n",
+    "            total_images,\n",
+    "            total_persons_count,\n",
+    "            images_no_persons,\n",
+    "            f\"{female_browsing_level_1_percentage:.2f}\",\n",
+    "            f\"{female_browsing_level_2_16_percentage:.2f}\",\n",
+    "            f\"{male_browsing_level_1_percentage:.2f}\",\n",
+    "            f\"{male_browsing_level_2_16_percentage:.2f}\",\n",
+    "            f\"{female_to_male_ratio:.2f}\" if female_to_male_ratio is not None else \"N/A\",\n",
+    "            f\"{female_age_stats['mean']:.2f} ({female_age_stats['std']:.2f})\",\n",
+    "            f\"{male_age_stats['mean']:.2f} ({male_age_stats['std']:.2f})\"\n",
+    "        ])\n",
+    "\n",
+    "# Sort table rows by the filename (assumes filenames are formatted with sortable dates)\n",
+    "table_rows = sorted(table_rows, key=lambda x: x[0])\n",
+    "\n",
+    "# Generate LaTeX table\n",
+    "latex_table = r\"\"\"\n",
+    "\\begin{table}[H]\n",
+    "\\centering\n",
+    "\\scriptsize\n",
+    "\\renewcommand{\\arraystretch}{0.9}\n",
+    "\\caption{Summary of Image Classification for 2023-2024}\n",
+    "\\label{table:image_classification_2023_2024}\n",
+    "\\begin{tabular}{lrrrrrrrrrr}\n",
+    "\\toprule\n",
+    "File Name & Total Images & Total Persons & No Persons & \\multicolumn{2}{c}{Female (\\%)} & \\multicolumn{2}{c}{Male (\\%)} & Female:Male & Female Age (Mean ± SD) & Male Age (Mean ± SD) \\\\\n",
+    " & & & & L1 & L2-16 & L1 & L2-16 & & & \\\\\n",
+    "\\midrule\n",
+    "\"\"\"\n",
+    "for row in table_rows:\n",
+    "    latex_table += \" & \".join(map(str, row)) + r\" \\\\\\\\\\n\"\n",
+    "\n",
+    "latex_table += r\"\"\"\n",
+    "\\bottomrule\n",
+    "\\end{tabular}\n",
+    "\\vspace{1em}\n",
+    "\\noindent\n",
+    "\\textbf{Disclaimer:} \\(\\female\\) and \\(\\male\\) refer to female-read and male-read classifications as determined by the MiVOLO system's weights. \n",
+    "We acknowledge the complexities of gender presentations and stress that these terms do not necessarily correspond to biological sex.\n",
+    "\\end{table}\n",
+    "\"\"\"\n",
+    "\n",
+    "# Output LaTeX table\n",
+    "print(\"\\nGenerated LaTeX Table:\")\n",
+    "print(latex_table)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ef428be-856b-4c4c-b1b0-a052d181d03b",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "HORDE",
+   "language": "python",
+   "name": "horde"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

scripts/Section_6-3_Models.ipynb

@@ -0,0 +1,210 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "14f6b6e3-5edb-458a-9553-09616455ba9f",
+   "metadata": {},
+   "source": [
+    "# Section 6.3: Download Models"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "64d0907d-780d-48a0-b2c4-a9de650a3760",
+   "metadata": {},
+   "source": [
+    "### Download models from CIVITAI to the respective folders to ~/ComfyUI/models\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2f788cc1-d8ff-43b8-9051-a3d5d8d79086",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T18:25:06.875713Z",
+     "iopub.status.busy": "2025-02-06T18:25:06.875579Z",
+     "iopub.status.idle": "2025-02-06T18:25:07.380814Z",
+     "shell.execute_reply": "2025-02-06T18:25:07.380184Z",
+     "shell.execute_reply.started": "2025-02-06T18:25:06.875698Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import re\n",
+    "import csv\n",
+    "import json\n",
+    "import time\n",
+    "import requests\n",
+    "from itertools import cycle\n",
+    "import pandas as pd\n",
+    "from requests.adapters import HTTPAdapter\n",
+    "from requests.packages.urllib3.util.retry import Retry\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "7b6349d2-b839-4b8e-895e-cc7eb3fba88f",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T18:25:07.848340Z",
+     "iopub.status.busy": "2025-02-06T18:25:07.848183Z",
+     "iopub.status.idle": "2025-02-06T18:25:07.851420Z",
+     "shell.execute_reply": "2025-02-06T18:25:07.850952Z",
+     "shell.execute_reply.started": "2025-02-06T18:25:07.848325Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "current_dir = Path.cwd()\n",
+    "api_karussell = current_dir.parent / 'misc/api_keys.txt'\n",
+    "target_dir = current_dir.parent / 'data/models/checkpoints/'  \n",
+    "target_dir.parent.mkdir(parents=True, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3c2a9244-5070-4159-821f-39bb92051daf",
+   "metadata": {},
+   "source": [
+    "## function definition:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "aeaaf1eb-9768-47ad-9298-7d7a8be59094",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T18:25:09.379460Z",
+     "iopub.status.busy": "2025-02-06T18:25:09.379303Z",
+     "iopub.status.idle": "2025-02-06T18:25:09.382034Z",
+     "shell.execute_reply": "2025-02-06T18:25:09.381557Z",
+     "shell.execute_reply.started": "2025-02-06T18:25:09.379446Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "csv_file_path = current_dir.parent / 'data/CSV/model_subsets/Civiverse_checkpoint_only.csv'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2cdc7051-af24-439c-b985-b5088bb13cc0",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T18:25:10.192171Z",
+     "iopub.status.busy": "2025-02-06T18:25:10.191997Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Downloaded: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/models/checkpoints/Pony/realDream_sdxlPony14.safetensors\n",
+      "Downloaded: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/models/checkpoints/Flux.1 D/acornIsSpinningFLUX_aisFluxV15.safetensors\n",
+      "Failed to download https://civitai.com/api/download/models/1376998: 401 Client Error: Unauthorized for url: https://civitai.com/api/download/models/1376998\n",
+      "Failed to download https://civitai.com/api/download/models/1379842: 401 Client Error: Unauthorized for url: https://civitai.com/api/download/models/1379842\n",
+      "Downloaded: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/models/checkpoints/Illustrious/miaomiaoHarem_v15a.safetensors\n"
+     ]
+    }
+   ],
+   "source": [
+    "with open(api_karussell, 'r') as f:\n",
+    "    api_keys = [line.strip() for line in f if line.strip()]\n",
+    "api_key_cycle = cycle(api_keys)  # Rotate API keys\n",
+    "\n",
+    "# Load CSV file\n",
+    "\n",
+    "df = pd.read_csv(csv_file_path)\n",
+    "\n",
+    "\n",
+    "\n",
+    "def get_filename_from_response(response, url):\n",
+    "    if 'content-disposition' in response.headers:\n",
+    "        content_disposition = response.headers['content-disposition']\n",
+    "        filename = content_disposition.split(\"filename=\")[-1].strip(\"\\\"\")\n",
+    "    else:\n",
+    "        filename = url.split(\"/\")[-1]\n",
+    "    return filename\n",
+    "\n",
+    "def download_file(url, dest_folder, api_key):\n",
+    "    headers = {'Authorization': f'Bearer {api_key}'}  # If authentication is needed\n",
+    "    \n",
+    "    try:\n",
+    "        response = requests.get(url, headers=headers, stream=True)\n",
+    "        response.raise_for_status()\n",
+    "        \n",
+    "        filename = get_filename_from_response(response, url)\n",
+    "        dest_path = dest_folder / filename\n",
+    "        \n",
+    "        with open(dest_path, 'wb') as f:\n",
+    "            for chunk in response.iter_content(chunk_size=8192):\n",
+    "                f.write(chunk)\n",
+    "        \n",
+    "        print(f\"Downloaded: {dest_path}\")\n",
+    "    except requests.exceptions.RequestException as e:\n",
+    "        print(f\"Failed to download {url}: {e}\")\n",
+    "\n",
+    "def main():\n",
+    "    for _, row in df.iterrows():\n",
+    "        url = row['downloadUrl']\n",
+    "        base_model = row['baseModel']\n",
+    "        \n",
+    "        if pd.isna(url) or pd.isna(base_model):\n",
+    "            continue  # Skip missing values\n",
+    "        \n",
+    "        # Create model-specific folder\n",
+    "        model_folder = target_dir / base_model\n",
+    "        model_folder.mkdir(parents=True, exist_ok=True)\n",
+    "        \n",
+    "        # Rotate API keys\n",
+    "        api_key = next(api_key_cycle)\n",
+    "        \n",
+    "        # Download filfile\n",
+    "        download_file(url, model_folder, api_key)\n",
+    "        \n",
+    "        # Sleep to avoid rate limits\n",
+    "        time.sleep(2)  # Adjust based on API limits\n",
+    "\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c99e2fee-eef2-4530-b69a-9a3db3bda84f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

scripts/Updated_Section_6-1_Models_ratios.ipynb

@@ -0,0 +1,547 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f743158b-c26f-417e-a7b6-1563d007eec4",
+   "metadata": {},
+   "source": [
+    "# Section 6.1 NSFW Escalation"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "106dfd44-1411-43ca-8753-8c79f5e01bad",
+   "metadata": {},
+   "source": [
+    "NSFW Ratio Calculation Workflow\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐  \n",
+    "│ Load Civiverse CSV   │ --> │ Read Civiverse dataset containing models,         │  \n",
+    "│                      │     │ metadata, and NSFW labels.                        │  \n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘  \n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐  \n",
+    "│ Expand Resource IDs  │ --> │ Extract resourceIDs from each model (IDs & hashes).│  \n",
+    "│                      │     │                                                   │  \n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘  \n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐  \n",
+    "│ Classify ResourceIDs │ --> │ Separate resourceIDs into ID-based and Hash-based │  \n",
+    "│ as 'ID' or 'Hash'    │     │ identifiers for matching.                         │  \n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘  \n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐  \n",
+    "│ Group by Resource ID │ --> │ Count occurrences of each ID/Hash and sum NSFW    │  \n",
+    "│ & Compute Matches    │     │ labels per resource.                              │  \n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘  \n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐  \n",
+    "│ Compute NSFW Ratio   │ --> │ NSFW Ratio = (NSFW True Count) / (Total Matches). │  \n",
+    "│                      │     │ Ensures fair assessment of NSFW presence.         │  \n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘  \n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐  \n",
+    "│ Merge Back to Models │ --> │ Attach calculated NSFW ratios to model dataset   │  \n",
+    "│ Dataset             │     │ for further analysis and visualization.           │  \n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘  \n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐  \n",
+    "│ Save Updated CSV     │ --> │ Write final dataset with NSFW ratios and metadata │  \n",
+    "│                      │     �� for structured storage and analysis.              │  \n",
+    "└──────────────────────┘     └───────────────────────────────────────────────────┘  \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9dc7445d",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:19:45.476339Z",
+     "iopub.status.busy": "2025-02-08T21:19:45.475634Z",
+     "iopub.status.idle": "2025-02-08T21:19:45.875103Z",
+     "shell.execute_reply": "2025-02-08T21:19:45.874435Z",
+     "shell.execute_reply.started": "2025-02-08T21:19:45.476313Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Paths initialized successfully.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Import necessary libraries\n",
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "import ast\n",
+    "import re\n",
+    "from multiprocessing import Pool, cpu_count\n",
+    "\n",
+    "# Define current working directory\n",
+    "current_dir = Path.cwd()\n",
+    "\n",
+    "# Define frequently used paths\n",
+    "data_dir = current_dir.parent / 'data/CSV'\n",
+    "subsets_dir = data_dir / 'model_subsets'\n",
+    "\n",
+    "# Input CSV files\n",
+    "civiverse_csv = data_dir / 'Civiverse.csv'\n",
+    "model_ids_csv = subsets_dir / 'model_ids.csv'\n",
+    "model_hashes_csv = subsets_dir / 'models_hashes.csv'\n",
+    "\n",
+    "# Output CSV file\n",
+    "models_output_csv = data_dir / 'Civiverse-Models_ratios.csv'\n",
+    "\n",
+    "# Ensure directories exist\n",
+    "subsets_dir.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "print('Paths initialized successfully.')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a417fd1d",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:19:45.921995Z",
+     "iopub.status.busy": "2025-02-08T21:19:45.921691Z",
+     "iopub.status.idle": "2025-02-08T21:19:45.924611Z",
+     "shell.execute_reply": "2025-02-08T21:19:45.924147Z",
+     "shell.execute_reply.started": "2025-02-08T21:19:45.921975Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "current_dir = Path.cwd()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "cd519fea",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:19:46.534207Z",
+     "iopub.status.busy": "2025-02-08T21:19:46.534034Z",
+     "iopub.status.idle": "2025-02-08T21:19:46.536779Z",
+     "shell.execute_reply": "2025-02-08T21:19:46.536283Z",
+     "shell.execute_reply.started": "2025-02-08T21:19:46.534191Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "civiverse = civiverse_csv\n",
+    "model_ids = model_ids_csv\n",
+    "model_hashes = model_hashes_csv"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1e8f462a-d6ba-4fec-95ff-a72273cc45c8",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-07T19:39:49.851283Z",
+     "iopub.status.busy": "2025-02-07T19:39:49.849959Z",
+     "iopub.status.idle": "2025-02-07T19:39:49.854079Z",
+     "shell.execute_reply": "2025-02-07T19:39:49.853522Z",
+     "shell.execute_reply.started": "2025-02-07T19:39:49.851229Z"
+    }
+   },
+   "source": [
+    "## Step 1 Match model IDs with Image NSFW (true/false)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "a14dd867",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:19:47.488085Z",
+     "iopub.status.busy": "2025-02-08T21:19:47.487915Z",
+     "iopub.status.idle": "2025-02-08T21:19:49.871306Z",
+     "shell.execute_reply": "2025-02-08T21:19:49.870619Z",
+     "shell.execute_reply.started": "2025-02-08T21:19:47.488069Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing chunk 0...\n",
+      "Chunk 0: Found 6860 hash entries\n",
+      "Processing chunk 1...\n",
+      "Chunk 1: Found 2282 hash entries\n",
+      "ID summary saved to /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/model_subsets/model_ids.csv\n",
+      "Hash summary saved to /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/model_subsets/models_hashes.csv\n"
+     ]
+    }
+   ],
+   "source": [
+    "model_ids.parent.mkdir(parents=True, exist_ok=True)\n",
+    "model_hashes.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Define a function to process each chunk\n",
+    "def process_chunk(chunk):\n",
+    "    chunk_subset = chunk[['generationSystem', 'resourceIDs', 'nsfw']].copy()\n",
+    "    chunk_subset['resourceIDs'] = chunk_subset['resourceIDs'].fillna('[]').astype(str)\n",
+    "\n",
+    "    # Expand the resourceIDs into individual rows\n",
+    "    chunk_subset['resourceIDs'] = chunk_subset['resourceIDs'].str.strip('[]').str.split(', ')\n",
+    "    expanded_chunk = chunk_subset.explode('resourceIDs')\n",
+    "\n",
+    "    # Clean resourceIDs\n",
+    "    expanded_chunk['resourceIDs'] = expanded_chunk['resourceIDs'].str.strip().str.strip(\"'\").str.strip('\"')\n",
+    "\n",
+    "    # Classify resources\n",
+    "    def classify_resource(resource):\n",
+    "        if pd.isna(resource):\n",
+    "            return None\n",
+    "        if resource.isdigit():\n",
+    "            return 'ID'\n",
+    "        elif re.fullmatch(r'[a-fA-F0-9]{8,64}', resource):  # Allow up to 64-char hashes\n",
+    "            return 'Hash'\n",
+    "        else:\n",
+    "            return None\n",
+    "\n",
+    "    expanded_chunk['Type'] = expanded_chunk['resourceIDs'].apply(classify_resource)\n",
+    "    expanded_chunk = expanded_chunk.dropna(subset=['Type'])\n",
+    "\n",
+    "    # Group data\n",
+    "    grouped_chunk = expanded_chunk.groupby(['resourceIDs', 'Type']).agg(\n",
+    "        matches=('resourceIDs', 'size'),\n",
+    "        nsfw_true=('nsfw', 'sum')\n",
+    "    ).reset_index()\n",
+    "    grouped_chunk['nsfw_ratio'] = grouped_chunk['nsfw_true'] / grouped_chunk['matches']\n",
+    "\n",
+    "    # Separate into ID and hash subsets\n",
+    "    ids_chunk = grouped_chunk[grouped_chunk['Type'] == 'ID'].drop(columns=['Type'])\n",
+    "    hashes_chunk = grouped_chunk[grouped_chunk['Type'] == 'Hash'].drop(columns=['Type'])\n",
+    "\n",
+    "    return ids_chunk, hashes_chunk\n",
+    "\n",
+    "# Write headers to output files\n",
+    "pd.DataFrame(columns=['ID', 'matches', 'nsfw_true', 'nsfw_ratio']).to_csv(model_ids, index=False)\n",
+    "pd.DataFrame(columns=['hash', 'matches', 'nsfw_true', 'nsfw_ratio']).to_csv(model_hashes, index=False)\n",
+    "\n",
+    "# Process the dataset in chunks\n",
+    "chunk_size = 100000\n",
+    "chunk_index = 0\n",
+    "\n",
+    "for chunk in pd.read_csv(civiverse, chunksize=chunk_size, low_memory=False):\n",
+    "    print(f\"Processing chunk {chunk_index}...\")\n",
+    "\n",
+    "    ids_chunk, hashes_chunk = process_chunk(chunk)\n",
+    "\n",
+    "    # Debugging to ensure hashes exist\n",
+    "    print(f\"Chunk {chunk_index}: Found {len(hashes_chunk)} hash entries\")\n",
+    "\n",
+    "    if len(hashes_chunk) == 0:\n",
+    "        print(\"Warning: No hash data in this chunk. Check classification function.\")\n",
+    "\n",
+    "    # Append data correctly\n",
+    "    ids_chunk.rename(columns={'resourceIDs': 'ID'}).to_csv(\n",
+    "        model_ids, index=False, mode='a', header=not model_ids.exists()\n",
+    "    )\n",
+    "    hashes_chunk.rename(columns={'resourceIDs': 'hash'}).to_csv(\n",
+    "        model_hashes, index=False, mode='a', header=not model_hashes.exists()\n",
+    "    )\n",
+    "\n",
+    "    chunk_index += 1\n",
+    "\n",
+    "print(f\"ID summary saved to {model_ids}\")\n",
+    "print(f\"Hash summary saved to {model_hashes}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "aa84573e-9d4f-4a1a-8db2-0c34244c2c3b",
+   "metadata": {},
+   "source": [
+    "## Step 2 Match IDs with Model-Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "7d28a287",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:19:49.872606Z",
+     "iopub.status.busy": "2025-02-08T21:19:49.872432Z",
+     "iopub.status.idle": "2025-02-08T21:19:50.072017Z",
+     "shell.execute_reply": "2025-02-08T21:19:50.071338Z",
+     "shell.execute_reply.started": "2025-02-08T21:19:49.872590Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Updated models dataset saved to /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/Civiverse-Models_ratios.csv\n",
+      "ID matches: 11\n",
+      "Version ID matches: 3\n",
+      "AutoV1 matches: 0\n",
+      "AutoV2 matches: 1\n",
+      "AutoV3 matches: 5\n",
+      "CRC32 matches: 0\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Paths to the main dataset and summary files\n",
+    "models_file_path = current_dir.parent / 'data/CSV/Civiverse-Models.csv'  # Replace with your models dataset path\n",
+    "ids_summary_file = model_ids_csv  # Replace with your IDs summary file path\n",
+    "hashes_summary_file = model_hashes_csv  # Replace with your hashes summary file path\n",
+    "models_output_file = models_output_csv\n",
+    "\n",
+    "# Load datasets\n",
+    "models_data = pd.read_csv(models_file_path)\n",
+    "ids_summary = pd.read_csv(ids_summary_file)\n",
+    "hashes_summary = pd.read_csv(hashes_summary_file)\n",
+    "\n",
+    "# Normalize column names\n",
+    "ids_summary.rename(columns={'ID': 'id'}, inplace=True)\n",
+    "hashes_summary.rename(columns={'hash': 'resourceID'}, inplace=True)\n",
+    "\n",
+    "# Ensure consistent data types\n",
+    "models_data['id'] = models_data['id'].astype(str).str.lower()\n",
+    "ids_summary['id'] = ids_summary['id'].astype(str).str.lower()\n",
+    "hash_columns = ['AutoV1', 'AutoV2', 'AutoV3', 'CRC32']\n",
+    "\n",
+    "# Normalize hash columns in models_data\n",
+    "for col in hash_columns:\n",
+    "    if col in models_data.columns:\n",
+    "        models_data[col] = models_data[col].astype(str).str.lower()\n",
+    "\n",
+    "# Flatten and clean hashes_summary\n",
+    "hashes_summary_cleaned = hashes_summary.groupby('resourceID').agg(\n",
+    "    matches=('matches', 'sum'),\n",
+    "    nsfw_true=('nsfw_true', 'sum'),\n",
+    "    nsfw_ratio=('nsfw_ratio', 'mean')\n",
+    ").reset_index()\n",
+    "\n",
+    "# Match IDs directly\n",
+    "models_data = models_data.merge(\n",
+    "    ids_summary,\n",
+    "    on='id',\n",
+    "    how='left',\n",
+    "    suffixes=('', '_id_summary')\n",
+    ")\n",
+    "models_data['id_matches'] = ~models_data['matches'].isna()\n",
+    "\n",
+    "# Match Version IDs\n",
+    "version_id_columns = [col for col in models_data.columns if col.startswith('version_id_')]\n",
+    "all_version_ids = ids_summary['id'].unique()  # All IDs from the summary for comparison\n",
+    "\n",
+    "# Create a flag for version ID matches\n",
+    "for col in version_id_columns:\n",
+    "    if col in models_data.columns:\n",
+    "        models_data[f'{col}_match'] = models_data[col].astype(str).str.lower().isin(all_version_ids)\n",
+    "\n",
+    "# Consolidate version ID matches\n",
+    "models_data['version_ids_match'] = models_data[[f'{col}_match' for col in version_id_columns]].any(axis=1)\n",
+    "\n",
+    "# Match Hashes\n",
+    "for column in hash_columns:\n",
+    "    if column in models_data.columns:\n",
+    "        models_data = models_data.merge(\n",
+    "            hashes_summary_cleaned,\n",
+    "            left_on=column,\n",
+    "            right_on='resourceID',\n",
+    "            how='left',\n",
+    "            suffixes=('', f'_{column}')\n",
+    "        )\n",
+    "        models_data[f'{column}_matches'] = ~models_data[f'matches_{column}'].isna()\n",
+    "        models_data['matches'] = models_data[['matches', f'matches_{column}']].sum(axis=1, skipna=True)\n",
+    "        models_data['nsfw_true'] = models_data[['nsfw_true', f'nsfw_true_{column}']].sum(axis=1, skipna=True)\n",
+    "        models_data.drop(columns=[f'matches_{column}', f'nsfw_true_{column}', f'nsfw_ratio_{column}', 'resourceID'], inplace=True)\n",
+    "\n",
+    "# Aggregate NSFW columns\n",
+    "models_data['nsfw_ratio'] = models_data['nsfw_true'] / models_data['matches']\n",
+    "models_data['nsfw_ratio'] = models_data['nsfw_ratio'].fillna(0)\n",
+    "\n",
+    "# Remove duplicates and keep the row with the highest `matches` and `nsfw_true`\n",
+    "models_data = models_data.sort_values(['id', 'matches', 'nsfw_true'], ascending=[True, False, False])\n",
+    "models_data = models_data.drop_duplicates(subset=['id'], keep='first')\n",
+    "\n",
+    "# Save updated dataset\n",
+    "models_data.to_csv(models_output_file, index=False)\n",
+    "print(f\"Updated models dataset saved to {models_output_file}\")\n",
+    "\n",
+    "# Verify match counts\n",
+    "id_match_count = models_data['id_matches'].sum()\n",
+    "version_id_match_count = models_data['version_ids_match'].sum()\n",
+    "hash_match_counts = {col: models_data[f'{col}_matches'].sum() for col in hash_columns if f'{col}_matches' in models_data}\n",
+    "\n",
+    "print(f\"ID matches: {id_match_count}\")\n",
+    "print(f\"Version ID matches: {version_id_match_count}\")\n",
+    "for col, count in hash_match_counts.items():\n",
+    "    print(f\"{col} matches: {count}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "76f7c8a5-64b7-4ee9-9099-cfdd34a91084",
+   "metadata": {},
+   "source": [
+    "## Step 3 get missing values from model preview images"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cb8cd4aa-a3de-47a8-b6f8-7227b10a92f7",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-07T20:31:23.202465Z",
+     "iopub.status.busy": "2025-02-07T20:31:23.202215Z",
+     "iopub.status.idle": "2025-02-07T20:31:23.220786Z",
+     "shell.execute_reply": "2025-02-07T20:31:23.219314Z",
+     "shell.execute_reply.started": "2025-02-07T20:31:23.202447Z"
+    }
+   },
+   "source": [
+    "this script reads the MODELS dataset from step 2, if there are 0 or less than 5 matches, it takes the civitai ID of the model, selectively calls the model API metadata and evaluates the NSFW level of the preview images for the model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "1887bc8d",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:19:51.442737Z",
+     "iopub.status.busy": "2025-02-08T21:19:51.442264Z",
+     "iopub.status.idle": "2025-02-08T21:23:37.409298Z",
+     "shell.execute_reply": "2025-02-08T21:23:37.408248Z",
+     "shell.execute_reply.started": "2025-02-08T21:19:51.442717Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Failed to fetch model 1219172: 404\n",
+      "Failed to fetch model 1221714: 404\n",
+      "Failed to fetch model 1224231: 404\n",
+      "Failed to fetch model 1224333: 404\n",
+      "Failed to fetch model 1224599: 404\n",
+      "Updated dataset saved to /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/models_ratios_a.csv\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/sctmp/lauwag/ipykernel_1495948/3660970368.py:54: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
+      "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
+      "\n",
+      "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
+      "\n",
+      "\n",
+      "  models_data[\"nsfw_ratio\"].fillna(0, inplace=True)  # Replace NaN with 0\n"
+     ]
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "\n",
+    "dataset_path = models_output_csv\n",
+    "\n",
+    "models_data = pd.read_csv(dataset_path)\n",
+    "api_base_url = \"https://civitai.com/api/v1/models/\"\n",
+    "output_dir = Path(current_dir.parent / 'data/raw/less_than_5_matches_models_temp/')\n",
+    "output_dir.mkdir(exist_ok=True)\n",
+    "\n",
+    "def fetch_model_data(model_id):\n",
+    "    \"\"\"Fetches model data from the API and saves it as a JSON file.\"\"\"\n",
+    "    file_path = output_dir / f\"model_{model_id}.json\"\n",
+    "    \n",
+    "    # Check if data already exists locally\n",
+    "    if file_path.exists():\n",
+    "        with open(file_path, \"r\", encoding=\"utf-8\") as f:\n",
+    "            return json.load(f)\n",
+    "    \n",
+    "    # Fetch data from the API\n",
+    "    url = f\"{api_base_url}{model_id}\"\n",
+    "    response = requests.get(url)\n",
+    "    if response.status_code == 200:\n",
+    "        data = response.json()\n",
+    "        with open(file_path, \"w\", encoding=\"utf-8\") as f:\n",
+    "            json.dump(data, f, indent=4)\n",
+    "        return data\n",
+    "    else:\n",
+    "        print(f\"Failed to fetch model {model_id}: {response.status_code}\")\n",
+    "        return None\n",
+    "\n",
+    "def process_model_json(data):\n",
+    "    \"\"\"Processes the JSON file to extract matches and NSFW values.\"\"\"\n",
+    "    nsfw_count = 0\n",
+    "    total_count = 0\n",
+    "    for version in data.get(\"modelVersions\", []):\n",
+    "        for image in version.get(\"images\", []):\n",
+    "            total_count += 1\n",
+    "            if image.get(\"nsfwLevel\", 1) > 1:\n",
+    "                nsfw_count += 1\n",
+    "    return total_count, nsfw_count\n",
+    "\n",
+    "# ✅ Now `models_data` is a DataFrame and `.iterrows()` works correctly\n",
+    "for index, row in models_data.iterrows():\n",
+    "    if row.get(\"matches\", 0) < 3:\n",
+    "        model_id = row[\"id\"]\n",
+    "        model_data = fetch_model_data(model_id)\n",
+    "        if model_data:\n",
+    "            total_images, nsfw_images = process_model_json(model_data)\n",
+    "            models_data.at[index, \"matches\"] = total_images\n",
+    "            models_data.at[index, \"nsfw_true\"] = nsfw_images\n",
+    "\n",
+    "# Recalculate nsfw_ratio\n",
+    "models_data[\"nsfw_ratio\"] = models_data[\"nsfw_true\"].div(models_data[\"matches\"], fill_value=0)\n",
+    "models_data[\"nsfw_ratio\"].fillna(0, inplace=True)  # Replace NaN with 0\n",
+    "\n",
+    "# Save updated dataset\n",
+    "output_file = current_dir.parent / 'data/CSV/models_ratios_a.csv'\n",
+    "models_data.to_csv(output_file, index=False)\n",
+    "print(f\"Updated dataset saved to {output_file}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6ec9ab6b",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}