Sarolanda commited on
Commit
5a277ae
Β·
1 Parent(s): b324a66

perf: servir fotos do disco local

Browse files
app.py CHANGED
@@ -18,7 +18,7 @@ from fastapi import Query
18
  from fastapi.staticfiles import StaticFiles
19
 
20
  from core.ai import AnimalAI
21
- from core.database import Database, DATA_DIR, PHOTOS_DIR
22
  from core.matcher import AnimalMatcher
23
  from core.seed import seed_if_empty
24
  from core.tracer import log_trace
@@ -53,8 +53,10 @@ _story_cache: dict[tuple[int, int], str] = {}
53
  app = Server()
54
 
55
  # Serve photos as static files at /photos/...
56
- PHOTOS_DIR.mkdir(parents=True, exist_ok=True)
57
- app.mount("/photos", StaticFiles(directory=str(PHOTOS_DIR)), name="photos")
 
 
58
 
59
  # Serve frontend assets (CSS, JS, images) at /static/...
60
  STATIC_DIR = Path(__file__).parent / "static"
 
18
  from fastapi.staticfiles import StaticFiles
19
 
20
  from core.ai import AnimalAI
21
+ from core.database import Database, DATA_DIR, PHOTOS_DIR, PHOTOS_CACHE
22
  from core.matcher import AnimalMatcher
23
  from core.seed import seed_if_empty
24
  from core.tracer import log_trace
 
53
  app = Server()
54
 
55
  # Serve photos as static files at /photos/...
56
+ # Servimos do cache local rapido (PHOTOS_CACHE), nao do bucket FUSE (lento).
57
+ # O bucket (PHOTOS_DIR) continua sendo a fonte de verdade para persistencia.
58
+ PHOTOS_CACHE.mkdir(parents=True, exist_ok=True)
59
+ app.mount("/photos", StaticFiles(directory=str(PHOTOS_CACHE)), name="photos")
60
 
61
  # Serve frontend assets (CSS, JS, images) at /static/...
62
  STATIC_DIR = Path(__file__).parent / "static"
core/__pycache__/__init__.cpython-310.pyc CHANGED
Binary files a/core/__pycache__/__init__.cpython-310.pyc and b/core/__pycache__/__init__.cpython-310.pyc differ
 
core/__pycache__/database.cpython-310.pyc CHANGED
Binary files a/core/__pycache__/database.cpython-310.pyc and b/core/__pycache__/database.cpython-310.pyc differ
 
core/database.py CHANGED
@@ -3,7 +3,9 @@ database.py β€” SQLite CRUD para animais e avistamentos.
3
  Usa /data/ em producao (HF Storage Bucket) ou ./data/ localmente.
4
  """
5
  import json
 
6
  import os
 
7
  import sqlite3
8
  import uuid
9
  from pathlib import Path
@@ -12,20 +14,52 @@ from typing import Optional
12
  import numpy as np
13
  from PIL import Image
14
 
 
 
15
  # ─── Paths ───────────────────────────────────────────────────────────────────
 
 
 
 
16
  _hf_data = Path("/data")
17
- DATA_DIR = _hf_data if (_hf_data.exists() and os.access(_hf_data, os.W_OK)) else Path("./data")
18
- DB_PATH = DATA_DIR / "viralata.db"
19
- PHOTOS_DIR = DATA_DIR / "photos"
20
- SCHEMA = Path(__file__).parent.parent / "db" / "schema.sql"
 
21
 
22
 
23
  class Database:
24
  def __init__(self):
25
  DATA_DIR.mkdir(parents=True, exist_ok=True)
26
  PHOTOS_DIR.mkdir(parents=True, exist_ok=True)
 
 
27
  self._init_db()
28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
  # ─── Internal ────────────────────────────────────────────────────────────
30
 
31
  def _conn(self) -> sqlite3.Connection:
@@ -86,7 +120,15 @@ class Database:
86
  subdir = PHOTOS_DIR / datetime.now().strftime("%Y-%m-%d")
87
  subdir.mkdir(parents=True, exist_ok=True)
88
  path = subdir / filename
89
- image.save(str(path), format="JPEG", quality=85)
 
 
 
 
 
 
 
 
90
  rel = path.relative_to(DATA_DIR)
91
  return str(rel).replace("\\", "/")
92
 
 
3
  Usa /data/ em producao (HF Storage Bucket) ou ./data/ localmente.
4
  """
5
  import json
6
+ import logging
7
  import os
8
+ import shutil
9
  import sqlite3
10
  import uuid
11
  from pathlib import Path
 
14
  import numpy as np
15
  from PIL import Image
16
 
17
+ log = logging.getLogger(__name__)
18
+
19
  # ─── Paths ───────────────────────────────────────────────────────────────────
20
+ # PHOTOS_DIR = fonte de verdade (bucket Xet montado em /data; lento por ser
21
+ # object storage sob demanda via FUSE).
22
+ # PHOTOS_CACHE = disco local efemero do Space (rapido). Servimos as imagens daqui;
23
+ # o bucket continua sendo onde gravamos para persistir entre restarts.
24
  _hf_data = Path("/data")
25
+ DATA_DIR = _hf_data if (_hf_data.exists() and os.access(_hf_data, os.W_OK)) else Path("./data")
26
+ DB_PATH = DATA_DIR / "viralata.db"
27
+ PHOTOS_DIR = DATA_DIR / "photos"
28
+ PHOTOS_CACHE = Path(os.environ.get("PHOTOS_CACHE_DIR", "/tmp/pawmap_photos"))
29
+ SCHEMA = Path(__file__).parent.parent / "db" / "schema.sql"
30
 
31
 
32
  class Database:
33
  def __init__(self):
34
  DATA_DIR.mkdir(parents=True, exist_ok=True)
35
  PHOTOS_DIR.mkdir(parents=True, exist_ok=True)
36
+ PHOTOS_CACHE.mkdir(parents=True, exist_ok=True)
37
+ self._warm_photo_cache()
38
  self._init_db()
39
 
40
+ # ─── Photo cache ───────────────────────────────────────────────────────────
41
+
42
+ def _warm_photo_cache(self):
43
+ """Copia as fotos do bucket (PHOTOS_DIR) para o disco local (PHOTOS_CACHE)
44
+ no boot, para que o serving nao dependa do mount FUSE lento."""
45
+ if PHOTOS_DIR.resolve() == PHOTOS_CACHE.resolve():
46
+ return
47
+ copied = 0
48
+ try:
49
+ for src in PHOTOS_DIR.rglob("*"):
50
+ if not src.is_file():
51
+ continue
52
+ dst = PHOTOS_CACHE / src.relative_to(PHOTOS_DIR)
53
+ if dst.exists() and dst.stat().st_size == src.stat().st_size:
54
+ continue
55
+ dst.parent.mkdir(parents=True, exist_ok=True)
56
+ shutil.copy2(src, dst)
57
+ copied += 1
58
+ except Exception as e:
59
+ log.warning("Falha ao aquecer cache de fotos: %s", e)
60
+ if copied:
61
+ log.info("Cache de fotos aquecido: %d arquivos copiados do bucket.", copied)
62
+
63
  # ─── Internal ────────────────────────────────────────────────────────────
64
 
65
  def _conn(self) -> sqlite3.Connection:
 
120
  subdir = PHOTOS_DIR / datetime.now().strftime("%Y-%m-%d")
121
  subdir.mkdir(parents=True, exist_ok=True)
122
  path = subdir / filename
123
+ image.save(str(path), format="JPEG", quality=85) # persiste no bucket
124
+ # Write-through: copia tambem pro cache local rapido (servido em /photos)
125
+ try:
126
+ rel_photo = path.relative_to(PHOTOS_DIR)
127
+ cache_path = PHOTOS_CACHE / rel_photo
128
+ cache_path.parent.mkdir(parents=True, exist_ok=True)
129
+ shutil.copy2(path, cache_path)
130
+ except Exception as e:
131
+ log.warning("Falha ao copiar foto pro cache: %s", e)
132
  rel = path.relative_to(DATA_DIR)
133
  return str(rel).replace("\\", "/")
134
 
core/seed.py CHANGED
@@ -144,15 +144,17 @@ def _random_embedding():
144
 
145
  def _copy_seed_photos():
146
  """Copia fotos de static/seed-photos/ (no repo) para DATA_DIR/photos/seed/ (runtime)."""
147
- from core.database import DATA_DIR, PHOTOS_DIR
148
  src_dir = Path(__file__).parent.parent / "static" / "seed-photos"
149
- dst_dir = PHOTOS_DIR / "seed"
150
- dst_dir.mkdir(parents=True, exist_ok=True)
151
- for src in src_dir.iterdir():
152
- dst = dst_dir / src.name
153
- if not dst.exists():
154
- shutil.copy2(src, dst)
155
- log.info("Foto copiada: %s", src.name)
 
 
156
 
157
 
158
  def seed_if_empty(db):
 
144
 
145
  def _copy_seed_photos():
146
  """Copia fotos de static/seed-photos/ (no repo) para DATA_DIR/photos/seed/ (runtime)."""
147
+ from core.database import PHOTOS_DIR, PHOTOS_CACHE
148
  src_dir = Path(__file__).parent.parent / "static" / "seed-photos"
149
+ # Copia para o bucket (persistencia) e para o cache local (serving rapido).
150
+ for base in (PHOTOS_DIR, PHOTOS_CACHE):
151
+ dst_dir = base / "seed"
152
+ dst_dir.mkdir(parents=True, exist_ok=True)
153
+ for src in src_dir.iterdir():
154
+ dst = dst_dir / src.name
155
+ if not dst.exists():
156
+ shutil.copy2(src, dst)
157
+ log.info("Foto copiada para %s: %s", base, src.name)
158
 
159
 
160
  def seed_if_empty(db):