From b2a5f85e4e1313c26c1a4ac275f042a6abf51225 Mon Sep 17 00:00:00 2001 From: Mattia Date: Sat, 30 Aug 2025 23:49:30 +0200 Subject: [PATCH] first setup --- README | 4 + access.log | 45 ++++++++ backend.py | 152 ++++++++++++++++++++++++++ backend.py.bacca | 88 +++++++++++++++ index.html | 276 +++++++++++++++++++++++++++++++++++++++++++++++ index.html.bacca | 114 ++++++++++++++++++++ tail_service.py | 66 ++++++++++++ 7 files changed, 745 insertions(+) create mode 100644 README create mode 100644 access.log create mode 100644 backend.py create mode 100644 backend.py.bacca create mode 100644 index.html create mode 100644 index.html.bacca create mode 100644 tail_service.py diff --git a/README b/README new file mode 100644 index 0000000..86ff4df --- /dev/null +++ b/README @@ -0,0 +1,4 @@ +# geolog +Tool to monitor the internet backgroung radiation. + +This readme is still to be done. diff --git a/access.log b/access.log new file mode 100644 index 0000000..5afa770 --- /dev/null +++ b/access.log @@ -0,0 +1,45 @@ +2025-06-01T08:15:23Z 192.0.2.1 GET /api/service1 +2025-06-01T08:17:45Z 203.0.113.5 POST /api/service2 +2025-06-01T08:18:10Z 198.51.100.42 GET /api/service3 +2025-06-01T09:02:13Z 8.8.8.8 GET /api/service1 +2025-06-01T09:15:55Z 1.1.1.1 POST /api/service2 +2025-06-01T09:20:37Z 45.33.32.156 GET /api/service3 +2025-06-02T11:05:11Z 91.198.174.192 GET /api/service1 +2025-06-02T11:07:28Z 151.101.65.69 GET /api/service2 +2025-06-02T11:09:50Z 104.244.42.1 GET /api/service3 +2025-06-02T11:22:17Z 185.199.108.153 GET /api/service1 +2025-06-03T14:01:01Z 203.0.113.25 POST /api/service1 +2025-06-03T14:05:33Z 192.0.2.50 GET /api/service2 +2025-06-03T14:15:10Z 203.0.113.77 GET /api/service3 +2025-06-04T16:45:20Z 8.8.4.4 GET /api/service1 +2025-06-04T16:50:55Z 1.0.0.1 POST /api/service2 +2025-06-04T16:55:40Z 208.67.222.222 GET /api/service3 +2025-06-05T10:10:10Z 208.67.220.220 GET /api/service1 +2025-06-05T10:20:45Z 64.233.160.0 GET /api/service2 +2025-06-05T10:30:30Z 17.172.224.47 GET /api/service3 +2025-06-06T12:00:00Z 52.95.110.1 GET /api/service1 +2025-06-06T12:05:05Z 13.107.246.45 GET /api/service2 +2025-06-06T12:15:15Z 40.90.22.1 GET /api/service3 +2025-06-07T18:30:25Z 23.45.67.89 GET /api/service1 +2025-06-07T18:35:40Z 34.56.78.90 GET /api/service2 +2025-06-07T18:40:55Z 56.78.90.12 GET /api/service3 +2025-06-08T21:10:05Z 66.249.64.1 GET /api/service1 +2025-06-08T21:15:30Z 172.217.10.78 GET /api/service2 +2025-06-08T21:20:45Z 74.125.224.72 GET /api/service3 +2025-06-09T06:25:12Z 157.240.20.35 GET /api/service1 +2025-06-09T06:30:33Z 31.13.71.36 GET /api/service2 +2025-06-09T06:40:40Z 69.63.176.13 GET /api/service3 +2025-06-10T13:00:00Z 203.0.113.111 GET /api/service1 +2025-06-10T13:05:20Z 192.0.2.200 GET /api/service2 +2025-06-10T13:10:30Z 203.0.113.222 GET /api/service3 +2025-06-11T09:45:50Z 91.121.0.1 GET /api/service1 +2025-06-11T09:55:15Z 185.60.216.35 GET /api/service2 +2025-06-11T10:05:30Z 216.58.214.14 GET /api/service3 +2025-06-12T15:15:15Z 13.107.21.200 GET /api/service1 +2025-06-12T15:20:20Z 40.112.72.205 GET /api/service2 +2025-06-12T15:30:30Z 52.95.245.1 GET /api/service3 +2025-06-12T15:30:30Z 52.95.245.1 GET /api/service3 +2025-06-12T15:30:30Z 52.95.245.1 GET /api/service3 +2025-06-12T15:30:30Z 52.95.245.1 GET /api/service3 +2025-08-30T20:39:54Z 93.70.82.245 POST /api/v4/users/status/ids +2025-08-30T20:40:01Z 82.60.174.5 POST /api/v4/users/status/ids diff --git a/backend.py b/backend.py new file mode 100644 index 0000000..f431c39 --- /dev/null +++ b/backend.py @@ -0,0 +1,152 @@ +import os +import pandas as pd +import geoip2.database +from fastapi import FastAPI, Query +from fastapi.middleware.cors import CORSMiddleware +from typing import Optional + +# ---------------------------- +# FastAPI Setup +# ---------------------------- +app = FastAPI(title="Reverse Proxy Connections Map API") + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"] +) + +# ---------------------------- +# GeoIP Setup +# ---------------------------- +reader = geoip2.database.Reader("GeoLite2-City.mmdb") +geo_cache = {} # cache IP lookups to save CPU + +def ip_to_geo(ip): + if ip in geo_cache: + return geo_cache[ip] + try: + response = reader.city(ip) + latlon = (response.location.latitude, response.location.longitude) + except Exception: + latlon = (None, None) + geo_cache[ip] = latlon + return latlon + +# ---------------------------- +# Helper: Parse timestamp from line +# ---------------------------- +def line_timestamp(line: str): + try: + ts_str = line.split(" ", 1)[0] + return pd.to_datetime(ts_str) + except Exception: + return None + +# ---------------------------- +# Binary search on lines +# ---------------------------- +def find_line_index(lines, target_time, seek_start=True): + lo, hi = 0, len(lines) - 1 + best_idx = None + + while lo <= hi: + mid = (lo + hi) // 2 + ts = line_timestamp(lines[mid]) + if ts is None: + # skip malformed line: move lo forward for start, hi backward for end + if seek_start: + lo = mid + 1 + else: + hi = mid - 1 + continue + + if seek_start: + if ts >= target_time: + best_idx = mid + hi = mid - 1 # search earlier + else: + lo = mid + 1 # search later + else: + if ts <= target_time: + best_idx = mid + lo = mid + 1 # search later + else: + hi = mid - 1 # search earlier + + # For end search, make sure we return the **last index ≤ target** + if best_idx is None: + return len(lines) - 1 if not seek_start else 0 + return best_idx + + + +# ---------------------------- +# Load logs using binary search on lines +# ---------------------------- +def load_logs_binary(service: Optional[str], start: Optional[str], end: Optional[str]): + start_dt = pd.to_datetime(start) if start else None + end_dt = pd.to_datetime(end) if end else None + records = [] + + with open("access.log", "r", errors="ignore") as f: + lines = f.readlines() + + start_idx = find_line_index(lines, start_dt, seek_start=True) if start_dt else 0 + end_idx = find_line_index(lines, end_dt, seek_start=False) if end_dt else len(lines) - 1 + + + for line in lines[start_idx:end_idx+1]: + try: + parts = line.strip().split(" ", 3) + if len(parts) != 4: + continue + timestamp, ip, method, path = parts + ts = pd.to_datetime(timestamp) + if start_dt and ts < start_dt: + continue + if end_dt and ts > end_dt: + break + if service and service not in path: + continue + lat, lon = ip_to_geo(ip) + if lat is None or lon is None: + continue + records.append({ + "timestamp": ts.isoformat(), + "ip": ip, + "path": path, + "lat": lat, + "lon": lon + }) + except Exception: + continue + + return records + +# ---------------------------- +# API Endpoint +# ---------------------------- +@app.get("/connections") +def get_connections( + service: Optional[str] = Query(None, description="Filter by service path"), + start: Optional[str] = Query(None, description="Start datetime in ISO format"), + end: Optional[str] = Query(None, description="End datetime in ISO format") +): + return load_logs_binary(service, start, end) + +# ---------------------------- +# Healthcheck +# ---------------------------- +@app.get("/health") +def health(): + size = os.path.getsize("access.log") + return {"status": "ok", "log_size_bytes": size, "cached_ips": len(geo_cache)} + +# ---------------------------- +# Run with Uvicorn +# ---------------------------- +if __name__ == "__main__": + import uvicorn + uvicorn.run("backend:app", host="0.0.0.0", port=8000, reload=True) diff --git a/backend.py.bacca b/backend.py.bacca new file mode 100644 index 0000000..a1d0c4d --- /dev/null +++ b/backend.py.bacca @@ -0,0 +1,88 @@ +# backend.py +from fastapi import FastAPI, Query +from fastapi.middleware.cors import CORSMiddleware +from typing import Optional +import pandas as pd +import geoip2.database +from datetime import datetime + +# ---------------------------- +# 1. Load Access Logs +# ---------------------------- +logs = [] +with open("access.log") as f: + for line in f: + # Example log format: "2025-08-28T12:34:56Z 192.0.2.1 GET /api/service1" + parts = line.strip().split(" ", 3) + if len(parts) != 4: + continue # skip malformed lines + timestamp, ip, method, path = parts + logs.append({ + "timestamp": timestamp, + "ip": ip, + "method": method, + "path": path + }) +df = pd.DataFrame(logs) +df["timestamp"] = pd.to_datetime(df["timestamp"]) +# ---------------------------- +# 2. GeoIP Lookup +# ---------------------------- +reader = geoip2.database.Reader("GeoLite2-City.mmdb") + +def ip_to_geo(ip): + try: + response = reader.city(ip) + return response.location.latitude, response.location.longitude + except Exception as e: + print(f"GeoIP lookup failed for {ip}: {e}") + return None, None +df["lat"], df["lon"] = zip(*df["ip"].apply(ip_to_geo)) +print(df) +df = df.dropna(subset=["lat", "lon"]) + + + +# ---------------------------- +# 3. FastAPI Setup +# ---------------------------- +app = FastAPI(title="Reverse Proxy Connections Map API") + +# Allow frontend to query API from any origin +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"] +) + +# ---------------------------- +# 4. API Endpoint +# ---------------------------- +@app.get("/connections") +def get_connections( + service: Optional[str] = Query(None, description="Filter by service path"), + start: Optional[str] = Query(None, description="Start datetime in ISO format"), + end: Optional[str] = Query(None, description="End datetime in ISO format") +): + data = df.copy() + + if service: + data = data[data["path"].str.contains(service)] + + if start: + data = data[data["timestamp"] >= pd.to_datetime(start)] + if end: + data = data[data["timestamp"] <= pd.to_datetime(end)] + return data[["timestamp", "path", "lat", "lon"]].to_dict(orient="records") + +# ---------------------------- +# 5. Healthcheck Endpoint +# ---------------------------- +@app.get("/health") +def health(): + return {"status": "ok", "total_connections": len(df)} + +if __name__ == "__main__": + import uvicorn + uvicorn.run("backend:app", host="0.0.0.0", port=8000, reload=True) diff --git a/index.html b/index.html new file mode 100644 index 0000000..d018c51 --- /dev/null +++ b/index.html @@ -0,0 +1,276 @@ + + + + + Connections Animation Map + + + + +
+ + + + + + + +
+ + +
+
+
+ + + + + diff --git a/index.html.bacca b/index.html.bacca new file mode 100644 index 0000000..854c179 --- /dev/null +++ b/index.html.bacca @@ -0,0 +1,114 @@ + + + + + Connections Animation Map + + + + +
+ + + + + +
+
+ + + + + diff --git a/tail_service.py b/tail_service.py new file mode 100644 index 0000000..598ec2b --- /dev/null +++ b/tail_service.py @@ -0,0 +1,66 @@ +mport re +import ipaddress +from datetime import datetime, timezone +import time + +ACCESS_LOG = "/var/log/nginx/access.log" +OUTPUT_LOG = "./file.log" + +INTERNAL_NETWORKS = [ + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("172.16.0.0/12"), +] + +log_line_re = re.compile( + r'(?P\S+) - - \[(?P