fabrica

hanna/fabrica

feat(blob): S3-compatible object storage for uploads

6b2bf21 · hanna committed on 2026-07-26

Add a `blob` crate exposing a `BlobStore` over the local filesystem or an
S3-compatible object store (AWS S3, MinIO, Cloudflare R2, Backblaze B2, …),
selected by `storage.backend = "local" | "s3"`. The `[storage.s3]` section
carries bucket, region, endpoint, credentials, path-style, and key-prefix, so
non-AWS stores work too. Bare git repos always stay local.

The S3 HTTP client is wired explicitly with the aws-lc-rs rustls provider so the
default (ring-based) client is never linked, keeping `ring` out of the tree and
the license gate green. Keys are namespaced and validated against traversal;
the local backend preserves the historical on-disk layout (no migration).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed by hannaSSH key fingerprint: SHA256:4g9cWhkAAw8gwqhJUcVbSnGEvdGwklW+1Aa/fMUu59k
8 files changed · +1314 −26UnifiedSplit
Cargo.lock +549 −21
@@ -306,13 +306,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
306checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"306checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
307307
308[[package]]308[[package]]
309name = "aws-credential-types"
310version = "1.2.10"
311source = "registry+https://github.com/rust-lang/crates.io-index"
312checksum = "b01c9521fa01558f750d183c8c68c81b0155b9d193a4ba7f84c36bd1b6d04a06"
313dependencies = [
314 "aws-smithy-async",
315 "aws-smithy-runtime-api",
316 "aws-smithy-types",
317 "zeroize",
318]
319
320[[package]]
309name = "aws-lc-rs"321name = "aws-lc-rs"
310version = "1.17.3"322version = "1.17.3"
311source = "registry+https://github.com/rust-lang/crates.io-index"323source = "registry+https://github.com/rust-lang/crates.io-index"
312checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1"324checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1"
313dependencies = [325dependencies = [
314 "aws-lc-sys",326 "aws-lc-sys",
315 "untrusted",327 "untrusted 0.7.1",
316 "zeroize",328 "zeroize",
317]329]
318330
@@ -330,6 +342,283 @@ dependencies = [
330]342]
331343
332[[package]]344[[package]]
345name = "aws-runtime"
346version = "1.5.16"
347source = "registry+https://github.com/rust-lang/crates.io-index"
348checksum = "7ce527fb7e53ba9626fc47824f25e256250556c40d8f81d27dd92aa38239d632"
349dependencies = [
350 "aws-credential-types",
351 "aws-sigv4",
352 "aws-smithy-async",
353 "aws-smithy-eventstream",
354 "aws-smithy-http",
355 "aws-smithy-runtime",
356 "aws-smithy-runtime-api",
357 "aws-smithy-types",
358 "aws-types",
359 "bytes",
360 "fastrand",
361 "http 0.2.12",
362 "http-body 0.4.6",
363 "percent-encoding",
364 "pin-project-lite",
365 "tracing",
366 "uuid",
367]
368
369[[package]]
370name = "aws-sdk-s3"
371version = "1.96.0"
372source = "registry+https://github.com/rust-lang/crates.io-index"
373checksum = "6e25d24de44b34dcdd5182ac4e4c6f07bcec2661c505acef94c0d293b65505fe"
374dependencies = [
375 "aws-credential-types",
376 "aws-runtime",
377 "aws-sigv4",
378 "aws-smithy-async",
379 "aws-smithy-checksums",
380 "aws-smithy-eventstream",
381 "aws-smithy-http",
382 "aws-smithy-json",
383 "aws-smithy-runtime",
384 "aws-smithy-runtime-api",
385 "aws-smithy-types",
386 "aws-smithy-xml",
387 "aws-types",
388 "bytes",
389 "fastrand",
390 "hex",
391 "hmac 0.12.1",
392 "http 0.2.12",
393 "http 1.4.2",
394 "http-body 0.4.6",
395 "lru",
396 "percent-encoding",
397 "regex-lite",
398 "sha2 0.10.9",
399 "tracing",
400 "url",
401]
402
403[[package]]
404name = "aws-sigv4"
405version = "1.3.6"
406source = "registry+https://github.com/rust-lang/crates.io-index"
407checksum = "c35452ec3f001e1f2f6db107b6373f1f48f05ec63ba2c5c9fa91f07dad32af11"
408dependencies = [
409 "aws-credential-types",
410 "aws-smithy-eventstream",
411 "aws-smithy-http",
412 "aws-smithy-runtime-api",
413 "aws-smithy-types",
414 "bytes",
415 "form_urlencoded",
416 "hex",
417 "hmac 0.12.1",
418 "http 0.2.12",
419 "http 1.4.2",
420 "percent-encoding",
421 "sha2 0.10.9",
422 "time",
423 "tracing",
424]
425
426[[package]]
427name = "aws-smithy-async"
428version = "1.2.6"
429source = "registry+https://github.com/rust-lang/crates.io-index"
430checksum = "127fcfad33b7dfc531141fda7e1c402ac65f88aca5511a4d31e2e3d2cd01ce9c"
431dependencies = [
432 "futures-util",
433 "pin-project-lite",
434 "tokio",
435]
436
437[[package]]
438name = "aws-smithy-checksums"
439version = "0.63.11"
440source = "registry+https://github.com/rust-lang/crates.io-index"
441checksum = "95bd108f7b3563598e4dc7b62e1388c9982324a2abd622442167012690184591"
442dependencies = [
443 "aws-smithy-http",
444 "aws-smithy-types",
445 "bytes",
446 "crc-fast",
447 "hex",
448 "http 0.2.12",
449 "http-body 0.4.6",
450 "md-5 0.10.6",
451 "pin-project-lite",
452 "sha1 0.10.7",
453 "sha2 0.10.9",
454 "tracing",
455]
456
457[[package]]
458name = "aws-smithy-eventstream"
459version = "0.60.13"
460source = "registry+https://github.com/rust-lang/crates.io-index"
461checksum = "e29a304f8319781a39808847efb39561351b1bb76e933da7aa90232673638658"
462dependencies = [
463 "aws-smithy-types",
464 "bytes",
465 "crc32fast",
466]
467
468[[package]]
469name = "aws-smithy-http"
470version = "0.62.5"
471source = "registry+https://github.com/rust-lang/crates.io-index"
472checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca"
473dependencies = [
474 "aws-smithy-eventstream",
475 "aws-smithy-runtime-api",
476 "aws-smithy-types",
477 "bytes",
478 "bytes-utils",
479 "futures-core",
480 "futures-util",
481 "http 0.2.12",
482 "http 1.4.2",
483 "http-body 0.4.6",
484 "percent-encoding",
485 "pin-project-lite",
486 "pin-utils",
487 "tracing",
488]
489
490[[package]]
491name = "aws-smithy-http-client"
492version = "1.1.4"
493source = "registry+https://github.com/rust-lang/crates.io-index"
494checksum = "623254723e8dfd535f566ee7b2381645f8981da086b5c4aa26c0c41582bb1d2c"
495dependencies = [
496 "aws-smithy-async",
497 "aws-smithy-runtime-api",
498 "aws-smithy-types",
499 "h2",
500 "http 1.4.2",
501 "hyper",
502 "hyper-rustls",
503 "hyper-util",
504 "pin-project-lite",
505 "rustls",
506 "rustls-native-certs",
507 "rustls-pki-types",
508 "tokio",
509 "tokio-rustls",
510 "tower",
511 "tracing",
512]
513
514[[package]]
515name = "aws-smithy-json"
516version = "0.61.7"
517source = "registry+https://github.com/rust-lang/crates.io-index"
518checksum = "2db31f727935fc63c6eeae8b37b438847639ec330a9161ece694efba257e0c54"
519dependencies = [
520 "aws-smithy-types",
521]
522
523[[package]]
524name = "aws-smithy-observability"
525version = "0.1.4"
526source = "registry+https://github.com/rust-lang/crates.io-index"
527checksum = "2d1881b1ea6d313f9890710d65c158bdab6fb08c91ea825f74c1c8c357baf4cc"
528dependencies = [
529 "aws-smithy-runtime-api",
530]
531
532[[package]]
533name = "aws-smithy-runtime"
534version = "1.9.4"
535source = "registry+https://github.com/rust-lang/crates.io-index"
536checksum = "0bbe9d018d646b96c7be063dd07987849862b0e6d07c778aad7d93d1be6c1ef0"
537dependencies = [
538 "aws-smithy-async",
539 "aws-smithy-http",
540 "aws-smithy-observability",
541 "aws-smithy-runtime-api",
542 "aws-smithy-types",
543 "bytes",
544 "fastrand",
545 "http 0.2.12",
546 "http 1.4.2",
547 "http-body 0.4.6",
548 "http-body 1.1.0",
549 "pin-project-lite",
550 "pin-utils",
551 "tokio",
552 "tracing",
553]
554
555[[package]]
556name = "aws-smithy-runtime-api"
557version = "1.9.2"
558source = "registry+https://github.com/rust-lang/crates.io-index"
559checksum = "ec7204f9fd94749a7c53b26da1b961b4ac36bf070ef1e0b94bb09f79d4f6c193"
560dependencies = [
561 "aws-smithy-async",
562 "aws-smithy-types",
563 "bytes",
564 "http 0.2.12",
565 "http 1.4.2",
566 "pin-project-lite",
567 "tokio",
568 "tracing",
569 "zeroize",
570]
571
572[[package]]
573name = "aws-smithy-types"
574version = "1.3.4"
575source = "registry+https://github.com/rust-lang/crates.io-index"
576checksum = "25f535879a207fce0db74b679cfc3e91a3159c8144d717d55f5832aea9eef46e"
577dependencies = [
578 "base64-simd",
579 "bytes",
580 "bytes-utils",
581 "futures-core",
582 "http 0.2.12",
583 "http 1.4.2",
584 "http-body 0.4.6",
585 "http-body 1.1.0",
586 "http-body-util",
587 "itoa",
588 "num-integer",
589 "pin-project-lite",
590 "pin-utils",
591 "ryu",
592 "serde",
593 "time",
594 "tokio",
595 "tokio-util",
596]
597
598[[package]]
599name = "aws-smithy-xml"
600version = "0.60.12"
601source = "registry+https://github.com/rust-lang/crates.io-index"
602checksum = "eab77cdd036b11056d2a30a7af7b775789fb024bf216acc13884c6c97752ae56"
603dependencies = [
604 "xmlparser",
605]
606
607[[package]]
608name = "aws-types"
609version = "1.3.10"
610source = "registry+https://github.com/rust-lang/crates.io-index"
611checksum = "d79fb68e3d7fe5d4833ea34dc87d2e97d26d3086cb3da660bb6b1f76d98680b6"
612dependencies = [
613 "aws-credential-types",
614 "aws-smithy-async",
615 "aws-smithy-runtime-api",
616 "aws-smithy-types",
617 "rustc_version",
618 "tracing",
619]
620
621[[package]]
333name = "axum"622name = "axum"
334version = "0.8.9"623version = "0.8.9"
335source = "registry+https://github.com/rust-lang/crates.io-index"624source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -339,8 +628,8 @@ dependencies = [
339 "bytes",628 "bytes",
340 "form_urlencoded",629 "form_urlencoded",
341 "futures-util",630 "futures-util",
342 "http",631 "http 1.4.2",
343 "http-body",632 "http-body 1.1.0",
344 "http-body-util",633 "http-body-util",
345 "hyper",634 "hyper",
346 "hyper-util",635 "hyper-util",
@@ -371,8 +660,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
371dependencies = [660dependencies = [
372 "bytes",661 "bytes",
373 "futures-core",662 "futures-core",
374 "http",663 "http 1.4.2",
375 "http-body",664 "http-body 1.1.0",
376 "http-body-util",665 "http-body-util",
377 "mime",666 "mime",
378 "pin-project-lite",667 "pin-project-lite",
@@ -394,8 +683,8 @@ dependencies = [
394 "cookie",683 "cookie",
395 "futures-core",684 "futures-core",
396 "futures-util",685 "futures-util",
397 "http",686 "http 1.4.2",
398 "http-body",687 "http-body 1.1.0",
399 "http-body-util",688 "http-body-util",
400 "mime",689 "mime",
401 "pin-project-lite",690 "pin-project-lite",
@@ -423,6 +712,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
423checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"712checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
424713
425[[package]]714[[package]]
715name = "base64-simd"
716version = "0.8.0"
717source = "registry+https://github.com/rust-lang/crates.io-index"
718checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
719dependencies = [
720 "outref",
721 "vsimd",
722]
723
724[[package]]
426name = "base64ct"725name = "base64ct"
427version = "1.8.3"726version = "1.8.3"
428source = "registry+https://github.com/rust-lang/crates.io-index"727source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -500,6 +799,19 @@ dependencies = [
500]799]
501800
502[[package]]801[[package]]
802name = "blob"
803version = "0.1.0"
804dependencies = [
805 "aws-sdk-s3",
806 "aws-smithy-http-client",
807 "bytes",
808 "config",
809 "tempfile",
810 "thiserror 2.0.19",
811 "tokio",
812]
813
814[[package]]
503name = "block-buffer"815name = "block-buffer"
504version = "0.10.4"816version = "0.10.4"
505source = "registry+https://github.com/rust-lang/crates.io-index"817source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -601,6 +913,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
601checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"913checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
602914
603[[package]]915[[package]]
916name = "bytes-utils"
917version = "0.1.4"
918source = "registry+https://github.com/rust-lang/crates.io-index"
919checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35"
920dependencies = [
921 "bytes",
922 "either",
923]
924
925[[package]]
604name = "bzip2"926name = "bzip2"
605version = "0.6.1"927version = "0.6.1"
606source = "registry+https://github.com/rust-lang/crates.io-index"928source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -972,6 +1294,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
972checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"1294checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
9731295
974[[package]]1296[[package]]
1297name = "crc-fast"
1298version = "1.6.0"
1299source = "registry+https://github.com/rust-lang/crates.io-index"
1300checksum = "6ddc2d09feefeee8bd78101665bd8645637828fa9317f9f292496dbbd8c65ff3"
1301dependencies = [
1302 "crc",
1303 "digest 0.10.7",
1304 "rand 0.9.5",
1305 "regex",
1306 "rustversion",
1307]
1308
1309[[package]]
975name = "crc24"1310name = "crc24"
976version = "0.1.6"1311version = "0.1.6"
977source = "registry+https://github.com/rust-lang/crates.io-index"1312source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1778,6 +2113,12 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
17782113
1779[[package]]2114[[package]]
1780name = "foldhash"2115name = "foldhash"
2116version = "0.1.5"
2117source = "registry+https://github.com/rust-lang/crates.io-index"
2118checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
2119
2120[[package]]
2121name = "foldhash"
1781version = "0.2.0"2122version = "0.2.0"
1782source = "registry+https://github.com/rust-lang/crates.io-index"2123source = "registry+https://github.com/rust-lang/crates.io-index"
1783checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"2124checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
@@ -2140,6 +2481,36 @@ dependencies = [
2140]2481]
21412482
2142[[package]]2483[[package]]
2484name = "h2"
2485version = "0.4.15"
2486source = "registry+https://github.com/rust-lang/crates.io-index"
2487checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
2488dependencies = [
2489 "atomic-waker",
2490 "bytes",
2491 "fnv",
2492 "futures-core",
2493 "futures-sink",
2494 "http 1.4.2",
2495 "indexmap",
2496 "slab",
2497 "tokio",
2498 "tokio-util",
2499 "tracing",
2500]
2501
2502[[package]]
2503name = "hashbrown"
2504version = "0.15.5"
2505source = "registry+https://github.com/rust-lang/crates.io-index"
2506checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
2507dependencies = [
2508 "allocator-api2",
2509 "equivalent",
2510 "foldhash 0.1.5",
2511]
2512
2513[[package]]
2143name = "hashbrown"2514name = "hashbrown"
2144version = "0.16.1"2515version = "0.16.1"
2145source = "registry+https://github.com/rust-lang/crates.io-index"2516source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2147,7 +2518,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
2147dependencies = [2518dependencies = [
2148 "allocator-api2",2519 "allocator-api2",
2149 "equivalent",2520 "equivalent",
2150 "foldhash",2521 "foldhash 0.2.0",
2151]2522]
21522523
2153[[package]]2524[[package]]
@@ -2250,6 +2621,17 @@ dependencies = [
22502621
2251[[package]]2622[[package]]
2252name = "http"2623name = "http"
2624version = "0.2.12"
2625source = "registry+https://github.com/rust-lang/crates.io-index"
2626checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
2627dependencies = [
2628 "bytes",
2629 "fnv",
2630 "itoa",
2631]
2632
2633[[package]]
2634name = "http"
2253version = "1.4.2"2635version = "1.4.2"
2254source = "registry+https://github.com/rust-lang/crates.io-index"2636source = "registry+https://github.com/rust-lang/crates.io-index"
2255checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"2637checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
@@ -2260,12 +2642,23 @@ dependencies = [
22602642
2261[[package]]2643[[package]]
2262name = "http-body"2644name = "http-body"
2645version = "0.4.6"
2646source = "registry+https://github.com/rust-lang/crates.io-index"
2647checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
2648dependencies = [
2649 "bytes",
2650 "http 0.2.12",
2651 "pin-project-lite",
2652]
2653
2654[[package]]
2655name = "http-body"
2263version = "1.1.0"2656version = "1.1.0"
2264source = "registry+https://github.com/rust-lang/crates.io-index"2657source = "registry+https://github.com/rust-lang/crates.io-index"
2265checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"2658checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
2266dependencies = [2659dependencies = [
2267 "bytes",2660 "bytes",
2268 "http",2661 "http 1.4.2",
2269]2662]
22702663
2271[[package]]2664[[package]]
@@ -2276,8 +2669,8 @@ checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
2276dependencies = [2669dependencies = [
2277 "bytes",2670 "bytes",
2278 "futures-core",2671 "futures-core",
2279 "http",2672 "http 1.4.2",
2280 "http-body",2673 "http-body 1.1.0",
2281 "pin-project-lite",2674 "pin-project-lite",
2282]2675]
22832676
@@ -2315,8 +2708,9 @@ dependencies = [
2315 "bytes",2708 "bytes",
2316 "futures-channel",2709 "futures-channel",
2317 "futures-core",2710 "futures-core",
2318 "http",2711 "h2",
2319 "http-body",2712 "http 1.4.2",
2713 "http-body 1.1.0",
2320 "httparse",2714 "httparse",
2321 "httpdate",2715 "httpdate",
2322 "itoa",2716 "itoa",
@@ -2327,6 +2721,22 @@ dependencies = [
2327]2721]
23282722
2329[[package]]2723[[package]]
2724name = "hyper-rustls"
2725version = "0.27.9"
2726source = "registry+https://github.com/rust-lang/crates.io-index"
2727checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
2728dependencies = [
2729 "http 1.4.2",
2730 "hyper",
2731 "hyper-util",
2732 "rustls",
2733 "rustls-native-certs",
2734 "tokio",
2735 "tokio-rustls",
2736 "tower-service",
2737]
2738
2739[[package]]
2330name = "hyper-tls"2740name = "hyper-tls"
2331version = "0.6.0"2741version = "0.6.0"
2332source = "registry+https://github.com/rust-lang/crates.io-index"2742source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2352,8 +2762,8 @@ dependencies = [
2352 "bytes",2762 "bytes",
2353 "futures-channel",2763 "futures-channel",
2354 "futures-util",2764 "futures-util",
2355 "http",2765 "http 1.4.2",
2356 "http-body",2766 "http-body 1.1.0",
2357 "hyper",2767 "hyper",
2358 "ipnet",2768 "ipnet",
2359 "libc",2769 "libc",
@@ -2787,6 +3197,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
2787checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"3197checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
27883198
2789[[package]]3199[[package]]
3200name = "lru"
3201version = "0.12.5"
3202source = "registry+https://github.com/rust-lang/crates.io-index"
3203checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
3204dependencies = [
3205 "hashbrown 0.15.5",
3206]
3207
3208[[package]]
2790name = "mail"3209name = "mail"
2791version = "0.1.0"3210version = "0.1.0"
2792dependencies = [3211dependencies = [
@@ -2835,7 +3254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
2835checksum = "8156733e27020ea5c684db5beac5d1d611e1272ab17901a49466294b84fc217e"3254checksum = "8156733e27020ea5c684db5beac5d1d611e1272ab17901a49466294b84fc217e"
2836dependencies = [3255dependencies = [
2837 "axum-core",3256 "axum-core",
2838 "http",3257 "http 1.4.2",
2839 "itoa",3258 "itoa",
2840 "maud_macros",3259 "maud_macros",
2841]3260]
@@ -2999,7 +3418,7 @@ dependencies = [
2999 "bytes",3418 "bytes",
3000 "encoding_rs",3419 "encoding_rs",
3001 "futures-util",3420 "futures-util",
3002 "http",3421 "http 1.4.2",
3003 "httparse",3422 "httparse",
3004 "memchr",3423 "memchr",
3005 "mime",3424 "mime",
@@ -3217,6 +3636,12 @@ dependencies = [
3217]3636]
32183637
3219[[package]]3638[[package]]
3639name = "outref"
3640version = "0.5.2"
3641source = "registry+https://github.com/rust-lang/crates.io-index"
3642checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
3643
3644[[package]]
3220name = "p256"3645name = "p256"
3221version = "0.13.2"3646version = "0.13.2"
3222source = "registry+https://github.com/rust-lang/crates.io-index"3647source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3547,6 +3972,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3547checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"3972checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
35483973
3549[[package]]3974[[package]]
3975name = "pin-utils"
3976version = "0.1.0"
3977source = "registry+https://github.com/rust-lang/crates.io-index"
3978checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
3979
3980[[package]]
3550name = "pkcs1"3981name = "pkcs1"
3551version = "0.7.5"3982version = "0.7.5"
3552source = "registry+https://github.com/rust-lang/crates.io-index"3983source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3882,6 +4313,12 @@ dependencies = [
3882]4313]
38834314
3884[[package]]4315[[package]]
4316name = "regex-lite"
4317version = "0.1.9"
4318source = "registry+https://github.com/rust-lang/crates.io-index"
4319checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
4320
4321[[package]]
3885name = "regex-syntax"4322name = "regex-syntax"
3886version = "0.8.11"4323version = "0.8.11"
3887source = "registry+https://github.com/rust-lang/crates.io-index"4324source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3902,8 +4339,8 @@ dependencies = [
3902 "base64",4339 "base64",
3903 "bytes",4340 "bytes",
3904 "futures-core",4341 "futures-core",
3905 "http",4342 "http 1.4.2",
3906 "http-body",4343 "http-body 1.1.0",
3907 "http-body-util",4344 "http-body-util",
3908 "hyper",4345 "hyper",
3909 "hyper-tls",4346 "hyper-tls",
@@ -3950,6 +4387,20 @@ dependencies = [
3950]4387]
39514388
3952[[package]]4389[[package]]
4390name = "ring"
4391version = "0.17.14"
4392source = "registry+https://github.com/rust-lang/crates.io-index"
4393checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
4394dependencies = [
4395 "cc",
4396 "cfg-if",
4397 "getrandom 0.2.17",
4398 "libc",
4399 "untrusted 0.9.0",
4400 "windows-sys 0.52.0",
4401]
4402
4403[[package]]
3953name = "ripemd"4404name = "ripemd"
3954version = "0.1.3"4405version = "0.1.3"
3955source = "registry+https://github.com/rust-lang/crates.io-index"4406source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4179,6 +4630,32 @@ dependencies = [
4179]4630]
41804631
4181[[package]]4632[[package]]
4633name = "rustls"
4634version = "0.23.42"
4635source = "registry+https://github.com/rust-lang/crates.io-index"
4636checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
4637dependencies = [
4638 "aws-lc-rs",
4639 "once_cell",
4640 "rustls-pki-types",
4641 "rustls-webpki",
4642 "subtle",
4643 "zeroize",
4644]
4645
4646[[package]]
4647name = "rustls-native-certs"
4648version = "0.8.4"
4649source = "registry+https://github.com/rust-lang/crates.io-index"
4650checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
4651dependencies = [
4652 "openssl-probe",
4653 "rustls-pki-types",
4654 "schannel",
4655 "security-framework",
4656]
4657
4658[[package]]
4182name = "rustls-pki-types"4659name = "rustls-pki-types"
4183version = "1.15.1"4660version = "1.15.1"
4184source = "registry+https://github.com/rust-lang/crates.io-index"4661source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4188,6 +4665,18 @@ dependencies = [
4188]4665]
41894666
4190[[package]]4667[[package]]
4668name = "rustls-webpki"
4669version = "0.103.13"
4670source = "registry+https://github.com/rust-lang/crates.io-index"
4671checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
4672dependencies = [
4673 "aws-lc-rs",
4674 "ring",
4675 "rustls-pki-types",
4676 "untrusted 0.9.0",
4677]
4678
4679[[package]]
4191name = "rustversion"4680name = "rustversion"
4192version = "1.0.23"4681version = "1.0.23"
4193source = "registry+https://github.com/rust-lang/crates.io-index"4682source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5221,6 +5710,16 @@ dependencies = [
5221]5710]
52225711
5223[[package]]5712[[package]]
5713name = "tokio-rustls"
5714version = "0.26.4"
5715source = "registry+https://github.com/rust-lang/crates.io-index"
5716checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
5717dependencies = [
5718 "rustls",
5719 "tokio",
5720]
5721
5722[[package]]
5224name = "tokio-stream"5723name = "tokio-stream"
5225version = "0.1.19"5724version = "0.1.19"
5226source = "registry+https://github.com/rust-lang/crates.io-index"5725source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5240,6 +5739,7 @@ dependencies = [
5240 "bytes",5739 "bytes",
5241 "futures-core",5740 "futures-core",
5242 "futures-sink",5741 "futures-sink",
5742 "libc",
5243 "pin-project-lite",5743 "pin-project-lite",
5244 "tokio",5744 "tokio",
5245]5745]
@@ -5312,8 +5812,8 @@ dependencies = [
5312 "bytes",5812 "bytes",
5313 "futures-core",5813 "futures-core",
5314 "futures-util",5814 "futures-util",
5315 "http",5815 "http 1.4.2",
5316 "http-body",5816 "http-body 1.1.0",
5317 "pin-project-lite",5817 "pin-project-lite",
5318 "tokio",5818 "tokio",
5319 "tokio-util",5819 "tokio-util",
@@ -5560,6 +6060,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
5560checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"6060checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
55616061
5562[[package]]6062[[package]]
6063name = "untrusted"
6064version = "0.9.0"
6065source = "registry+https://github.com/rust-lang/crates.io-index"
6066checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
6067
6068[[package]]
5563name = "url"6069name = "url"
5564version = "2.5.8"6070version = "2.5.8"
5565source = "registry+https://github.com/rust-lang/crates.io-index"6071source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5619,6 +6125,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
5619checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"6125checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
56206126
5621[[package]]6127[[package]]
6128name = "vsimd"
6129version = "0.8.0"
6130source = "registry+https://github.com/rust-lang/crates.io-index"
6131checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
6132
6133[[package]]
5622name = "walkdir"6134name = "walkdir"
5623version = "2.5.0"6135version = "2.5.0"
5624source = "registry+https://github.com/rust-lang/crates.io-index"6136source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5717,6 +6229,7 @@ dependencies = [
5717 "axum",6229 "axum",
5718 "axum-extra",6230 "axum-extra",
5719 "base64",6231 "base64",
6232 "blob",
5720 "comrak",6233 "comrak",
5721 "config",6234 "config",
5722 "flate2",6235 "flate2",
@@ -5898,6 +6411,15 @@ dependencies = [
58986411
5899[[package]]6412[[package]]
5900name = "windows-sys"6413name = "windows-sys"
6414version = "0.52.0"
6415source = "registry+https://github.com/rust-lang/crates.io-index"
6416checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
6417dependencies = [
6418 "windows-targets",
6419]
6420
6421[[package]]
6422name = "windows-sys"
5901version = "0.59.0"6423version = "0.59.0"
5902source = "registry+https://github.com/rust-lang/crates.io-index"6424source = "registry+https://github.com/rust-lang/crates.io-index"
5903checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"6425checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
@@ -6041,6 +6563,12 @@ dependencies = [
6041]6563]
60426564
6043[[package]]6565[[package]]
6566name = "xmlparser"
6567version = "0.13.6"
6568source = "registry+https://github.com/rust-lang/crates.io-index"
6569checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
6570
6571[[package]]
6044name = "yansi"6572name = "yansi"
6045version = "1.0.1"6573version = "1.0.1"
6046source = "registry+https://github.com/rust-lang/crates.io-index"6574source = "registry+https://github.com/rust-lang/crates.io-index"
Cargo.toml +15 −0
@@ -55,6 +55,7 @@ publish = false
55config = { path = "crates/config" }55config = { path = "crates/config" }
56store = { path = "crates/store" }56store = { path = "crates/store" }
57model = { path = "crates/model" }57model = { path = "crates/model" }
58blob = { path = "crates/blob" }
58git = { path = "crates/git" }59git = { path = "crates/git" }
59highlight = { path = "crates/highlight" }60highlight = { path = "crates/highlight" }
60auth = { path = "crates/auth" }61auth = { path = "crates/auth" }
@@ -173,6 +174,20 @@ reqwest = { version = "0.12", default-features = false, features = [
173] }174] }
174# SSH server (default features: aws-lc-rs backend, no ring; re-exports ssh-key).175# SSH server (default features: aws-lc-rs backend, no ring; re-exports ssh-key).
175russh = "0.62"176russh = "0.62"
177# S3-compatible object storage for uploads (avatars, release assets, LFS).
178# aws-lc-rs crypto (already in the tree via russh) so `ring` stays out and the
179# license gate holds; see docs/decisions.md. Manual client config (static creds,
180# custom endpoint) avoids the heavier aws-config credential-provider chain.
181aws-sdk-s3 = { version = "1", default-features = false, features = [
182 "behavior-version-latest",
183 "rt-tokio",
184] }
185# The HTTP client is wired explicitly with the aws-lc-rs rustls provider so the
186# default (ring-based) client is never pulled in — keeping `ring` out of the tree.
187aws-smithy-http-client = { version = "1", default-features = false, features = [
188 "rustls-aws-lc",
189] }
190bytes = "1"
176tracing = "0.1"191tracing = "0.1"
177tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "json"] }192tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "json"] }
178193
crates/blob/Cargo.toml +24 −0
@@ -0,0 +1,24 @@
1[package]
2name = "blob"
3description = "Upload storage over the local filesystem or an S3-compatible object store."
4version.workspace = true
5edition.workspace = true
6rust-version.workspace = true
7license.workspace = true
8authors.workspace = true
9repository.workspace = true
10publish.workspace = true
11
12[dependencies]
13config = { workspace = true }
14tokio = { workspace = true }
15thiserror = { workspace = true }
16aws-sdk-s3 = { workspace = true }
17aws-smithy-http-client = { workspace = true }
18bytes = { workspace = true }
19
20[dev-dependencies]
21tempfile = { workspace = true }
22
23[lints]
24workspace = true
crates/blob/src/lib.rs +562 −0
@@ -0,0 +1,562 @@
1// This Source Code Form is subject to the terms of the Mozilla Public
2// License, v. 2.0. If a copy of the MPL was not distributed with this
3// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
5//! Upload storage: one abstraction over the local filesystem and S3-compatible
6//! object stores.
7//!
8//! Bare git repositories always live on local disk (git needs filesystem
9//! access); only *uploads* — user avatars, release assets, and git-LFS objects —
10//! flow through here, so an operator can offload them to S3, `MinIO`, R2, or any
11//! compatible store by setting `storage.backend = "s3"`.
12//!
13//! Objects are addressed by a [`Namespace`] plus a key. The local backend maps a
14//! namespace to its on-disk directory (preserving the historical layout); the S3
15//! backend maps it to a key prefix. Keys are validated to forbid path traversal.
16
17use std::path::{Path, PathBuf};
18use std::pin::Pin;
19
20use config::{S3, Storage, StorageBackend};
21use tokio::io::{AsyncRead, AsyncWriteExt as _};
22
23/// A logical group of uploads, mapped to a directory (local) or key prefix (S3).
24#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub enum Namespace {
26 /// User profile avatars, keyed by user id.
27 Avatars,
28 /// Release asset files, keyed by asset id.
29 Releases,
30 /// git-LFS objects, keyed by their sharded sha256 oid.
31 Lfs,
32}
33
34impl Namespace {
35 /// The directory / key-prefix segment for this namespace.
36 fn segment(self) -> &'static str {
37 match self {
38 Namespace::Avatars => "avatars",
39 Namespace::Releases => "releases",
40 Namespace::Lfs => "lfs",
41 }
42 }
43}
44
45/// Errors from the blob store.
46#[derive(Debug, thiserror::Error)]
47pub enum BlobError {
48 /// A local filesystem error.
49 #[error("blob io error: {0}")]
50 Io(#[from] std::io::Error),
51 /// An object-store (S3) error, stringified from the SDK.
52 #[error("object store error: {0}")]
53 Backend(String),
54 /// The key contained an empty, `.`, `..`, or absolute segment.
55 #[error("invalid object key")]
56 InvalidKey,
57 /// `backend = "s3"` without a `[storage.s3]` section.
58 #[error("s3 backend requires the [storage.s3] configuration")]
59 MissingS3Config,
60}
61
62/// A readable object: its byte length plus an async reader over its contents.
63pub struct BlobRead {
64 /// The object's size in bytes.
65 pub size: u64,
66 /// A streaming reader over the object.
67 pub reader: Pin<Box<dyn AsyncRead + Send>>,
68}
69
70/// The upload store, over either the local filesystem or an S3-compatible store.
71#[derive(Clone)]
72pub enum BlobStore {
73 /// Local filesystem backend.
74 Local(LocalStore),
75 /// S3-compatible object-store backend.
76 S3(S3Store),
77}
78
79/// The local filesystem backend: one root directory per namespace.
80#[derive(Clone)]
81pub struct LocalStore {
82 avatars: PathBuf,
83 releases: PathBuf,
84 lfs: PathBuf,
85}
86
87/// The S3-compatible backend.
88#[derive(Clone)]
89pub struct S3Store {
90 client: aws_sdk_s3::Client,
91 bucket: String,
92 prefix: String,
93}
94
95impl BlobStore {
96 /// Build the store described by `[storage]`. The local backend reuses the
97 /// existing on-disk directories; the S3 backend builds a client from
98 /// `[storage.s3]` with static credentials and the configured endpoint.
99 ///
100 /// # Errors
101 ///
102 /// Returns [`BlobError::MissingS3Config`] if `backend = "s3"` but no
103 /// `[storage.s3]` section is present.
104 pub fn from_config(storage: &Storage) -> Result<Self, BlobError> {
105 match storage.backend {
106 StorageBackend::Local => Ok(BlobStore::Local(LocalStore {
107 avatars: storage.data_dir.join("avatars"),
108 releases: storage.data_dir.join("releases"),
109 lfs: storage.lfs_dir.clone(),
110 })),
111 StorageBackend::S3 => {
112 let s3 = storage.s3.as_ref().ok_or(BlobError::MissingS3Config)?;
113 Ok(BlobStore::S3(S3Store {
114 client: build_s3_client(s3),
115 bucket: s3.bucket.clone(),
116 prefix: s3.prefix.clone(),
117 }))
118 }
119 }
120 }
121
122 /// Move the file at `src` into the store at `(namespace, key)`, consuming it
123 /// (the source is removed on success). Streams for the S3 backend.
124 ///
125 /// # Errors
126 ///
127 /// [`BlobError::InvalidKey`] for an unsafe key; otherwise a backend error.
128 pub async fn put_file(
129 &self,
130 namespace: Namespace,
131 key: &str,
132 src: &Path,
133 content_type: &str,
134 ) -> Result<(), BlobError> {
135 check_key(key)?;
136 match self {
137 BlobStore::Local(local) => {
138 move_in(src, &local.path(namespace, key)).await?;
139 Ok(())
140 }
141 BlobStore::S3(s3) => {
142 let body = aws_sdk_s3::primitives::ByteStream::from_path(src)
143 .await
144 .map_err(|e| BlobError::Backend(e.to_string()))?;
145 s3.client
146 .put_object()
147 .bucket(&s3.bucket)
148 .key(s3.key(namespace, key))
149 .content_type(content_type)
150 .body(body)
151 .send()
152 .await
153 .map_err(|e| BlobError::Backend(e.to_string()))?;
154 tokio::fs::remove_file(src).await?;
155 Ok(())
156 }
157 }
158 }
159
160 /// Store `bytes` at `(namespace, key)`, replacing any existing object.
161 ///
162 /// # Errors
163 ///
164 /// [`BlobError::InvalidKey`] for an unsafe key; otherwise a backend error.
165 pub async fn put_bytes(
166 &self,
167 namespace: Namespace,
168 key: &str,
169 bytes: Vec<u8>,
170 content_type: &str,
171 ) -> Result<(), BlobError> {
172 check_key(key)?;
173 match self {
174 BlobStore::Local(local) => {
175 let path = local.path(namespace, key);
176 if let Some(parent) = path.parent() {
177 tokio::fs::create_dir_all(parent).await?;
178 }
179 let mut file = tokio::fs::File::create(&path).await?;
180 file.write_all(&bytes).await?;
181 file.flush().await?;
182 Ok(())
183 }
184 BlobStore::S3(s3) => {
185 s3.client
186 .put_object()
187 .bucket(&s3.bucket)
188 .key(s3.key(namespace, key))
189 .content_type(content_type)
190 .body(bytes.into())
191 .send()
192 .await
193 .map_err(|e| BlobError::Backend(e.to_string()))?;
194 Ok(())
195 }
196 }
197 }
198
199 /// Read an entire object into memory, or `None` if it does not exist.
200 ///
201 /// # Errors
202 ///
203 /// [`BlobError::InvalidKey`] for an unsafe key; otherwise a backend error.
204 pub async fn get_bytes(
205 &self,
206 namespace: Namespace,
207 key: &str,
208 ) -> Result<Option<Vec<u8>>, BlobError> {
209 check_key(key)?;
210 match self {
211 BlobStore::Local(local) => match tokio::fs::read(local.path(namespace, key)).await {
212 Ok(bytes) => Ok(Some(bytes)),
213 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
214 Err(e) => Err(e.into()),
215 },
216 BlobStore::S3(s3) => match s3
217 .client
218 .get_object()
219 .bucket(&s3.bucket)
220 .key(s3.key(namespace, key))
221 .send()
222 .await
223 {
224 Ok(resp) => {
225 let data = resp
226 .body
227 .collect()
228 .await
229 .map_err(|e| BlobError::Backend(e.to_string()))?;
230 Ok(Some(data.to_vec()))
231 }
232 Err(e) => {
233 let svc = e.into_service_error();
234 if svc.is_no_such_key() {
235 Ok(None)
236 } else {
237 Err(BlobError::Backend(svc.to_string()))
238 }
239 }
240 },
241 }
242 }
243
244 /// Open an object for streaming, or `None` if it does not exist.
245 ///
246 /// # Errors
247 ///
248 /// [`BlobError::InvalidKey`] for an unsafe key; otherwise a backend error.
249 pub async fn open(
250 &self,
251 namespace: Namespace,
252 key: &str,
253 ) -> Result<Option<BlobRead>, BlobError> {
254 check_key(key)?;
255 match self {
256 BlobStore::Local(local) => {
257 match tokio::fs::File::open(local.path(namespace, key)).await {
258 Ok(file) => {
259 let size = file.metadata().await?.len();
260 Ok(Some(BlobRead {
261 size,
262 reader: Box::pin(file),
263 }))
264 }
265 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
266 Err(e) => Err(e.into()),
267 }
268 }
269 BlobStore::S3(s3) => match s3
270 .client
271 .get_object()
272 .bucket(&s3.bucket)
273 .key(s3.key(namespace, key))
274 .send()
275 .await
276 {
277 Ok(resp) => {
278 let size = u64::try_from(resp.content_length().unwrap_or(0)).unwrap_or(0);
279 Ok(Some(BlobRead {
280 size,
281 reader: Box::pin(resp.body.into_async_read()),
282 }))
283 }
284 Err(e) => {
285 let svc = e.into_service_error();
286 if svc.is_no_such_key() {
287 Ok(None)
288 } else {
289 Err(BlobError::Backend(svc.to_string()))
290 }
291 }
292 },
293 }
294 }
295
296 /// Delete an object. Missing objects are not an error (idempotent).
297 ///
298 /// # Errors
299 ///
300 /// [`BlobError::InvalidKey`] for an unsafe key; otherwise a backend error.
301 pub async fn delete(&self, namespace: Namespace, key: &str) -> Result<(), BlobError> {
302 check_key(key)?;
303 match self {
304 BlobStore::Local(local) => {
305 match tokio::fs::remove_file(local.path(namespace, key)).await {
306 Ok(()) => Ok(()),
307 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
308 Err(e) => Err(e.into()),
309 }
310 }
311 BlobStore::S3(s3) => {
312 s3.client
313 .delete_object()
314 .bucket(&s3.bucket)
315 .key(s3.key(namespace, key))
316 .send()
317 .await
318 .map_err(|e| BlobError::Backend(e.to_string()))?;
319 Ok(())
320 }
321 }
322 }
323
324 /// Whether an object exists.
325 ///
326 /// # Errors
327 ///
328 /// [`BlobError::InvalidKey`] for an unsafe key; otherwise a backend error.
329 pub async fn exists(&self, namespace: Namespace, key: &str) -> Result<bool, BlobError> {
330 check_key(key)?;
331 match self {
332 BlobStore::Local(local) => Ok(tokio::fs::try_exists(local.path(namespace, key)).await?),
333 BlobStore::S3(s3) => match s3
334 .client
335 .head_object()
336 .bucket(&s3.bucket)
337 .key(s3.key(namespace, key))
338 .send()
339 .await
340 {
341 Ok(_) => Ok(true),
342 Err(e) => {
343 let svc = e.into_service_error();
344 if svc.is_not_found() {
345 Ok(false)
346 } else {
347 Err(BlobError::Backend(svc.to_string()))
348 }
349 }
350 },
351 }
352 }
353}
354
355impl LocalStore {
356 /// The on-disk path for `(namespace, key)`.
357 fn path(&self, namespace: Namespace, key: &str) -> PathBuf {
358 let root = match namespace {
359 Namespace::Avatars => &self.avatars,
360 Namespace::Releases => &self.releases,
361 Namespace::Lfs => &self.lfs,
362 };
363 root.join(key)
364 }
365}
366
367impl S3Store {
368 /// The object key for `(namespace, key)`: `{prefix}{segment}/{key}`.
369 fn key(&self, namespace: Namespace, key: &str) -> String {
370 format!("{}{}/{}", self.prefix, namespace.segment(), key)
371 }
372}
373
374/// Build an S3 client from static credentials and the configured endpoint. The
375/// HTTP client is wired explicitly with the aws-lc-rs rustls provider so the
376/// default (ring-based) client is never linked.
377fn build_s3_client(s3: &S3) -> aws_sdk_s3::Client {
378 use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
379 use aws_smithy_http_client::{Builder as HttpBuilder, tls};
380
381 let http = HttpBuilder::new()
382 .tls_provider(tls::Provider::Rustls(
383 tls::rustls_provider::CryptoMode::AwsLc,
384 ))
385 .build_https();
386 let credentials = Credentials::new(
387 s3.access_key_id.clone(),
388 s3.secret_access_key.expose().to_string(),
389 None,
390 None,
391 "fabrica-config",
392 );
393 let mut builder = aws_sdk_s3::Config::builder()
394 .behavior_version(BehaviorVersion::latest())
395 .http_client(http)
396 .region(Region::new(s3.region.clone()))
397 .credentials_provider(credentials)
398 .force_path_style(s3.path_style);
399 if let Some(endpoint) = &s3.endpoint {
400 builder = builder.endpoint_url(endpoint);
401 }
402 aws_sdk_s3::Client::from_conf(builder.build())
403}
404
405/// Move `src` to `dest`, creating parent directories, with a copy fallback if the
406/// two live on different filesystems (rename fails with `EXDEV`).
407async fn move_in(src: &Path, dest: &Path) -> std::io::Result<()> {
408 if let Some(parent) = dest.parent() {
409 tokio::fs::create_dir_all(parent).await?;
410 }
411 if tokio::fs::rename(src, dest).await.is_ok() {
412 return Ok(());
413 }
414 tokio::fs::copy(src, dest).await?;
415 tokio::fs::remove_file(src).await
416}
417
418/// Reject keys that could escape the namespace root: empty, absolute, or with a
419/// `.`/`..`/empty path segment.
420fn check_key(key: &str) -> Result<(), BlobError> {
421 if key.is_empty()
422 || key.starts_with('/')
423 || key
424 .split('/')
425 .any(|seg| seg.is_empty() || seg == "." || seg == "..")
426 {
427 return Err(BlobError::InvalidKey);
428 }
429 Ok(())
430}
431
432#[cfg(test)]
433mod tests {
434 #![allow(clippy::unwrap_used)]
435
436 use config::{Storage, StorageBackend};
437 use tokio::io::AsyncReadExt as _;
438
439 use super::*;
440
441 fn local_store(dir: &Path) -> BlobStore {
442 let storage = Storage {
443 data_dir: dir.to_path_buf(),
444 repo_dir: dir.join("repos"),
445 lfs_dir: dir.join("lfs"),
446 backend: StorageBackend::Local,
447 s3: None,
448 };
449 BlobStore::from_config(&storage).unwrap()
450 }
451
452 #[tokio::test]
453 async fn local_round_trip() {
454 let dir = tempfile::tempdir().unwrap();
455 let store = local_store(dir.path());
456
457 // Missing object.
458 assert!(!store.exists(Namespace::Avatars, "u1").await.unwrap());
459 assert!(
460 store
461 .get_bytes(Namespace::Avatars, "u1")
462 .await
463 .unwrap()
464 .is_none()
465 );
466 assert!(
467 store
468 .open(Namespace::Avatars, "u1")
469 .await
470 .unwrap()
471 .is_none()
472 );
473
474 // put_bytes / get_bytes / exists.
475 store
476 .put_bytes(Namespace::Avatars, "u1", b"hello".to_vec(), "image/png")
477 .await
478 .unwrap();
479 assert!(store.exists(Namespace::Avatars, "u1").await.unwrap());
480 assert_eq!(
481 store
482 .get_bytes(Namespace::Avatars, "u1")
483 .await
484 .unwrap()
485 .unwrap(),
486 b"hello"
487 );
488
489 // open streams the same bytes with the right size.
490 let mut read = store.open(Namespace::Avatars, "u1").await.unwrap().unwrap();
491 assert_eq!(read.size, 5);
492 let mut buf = Vec::new();
493 read.reader.read_to_end(&mut buf).await.unwrap();
494 assert_eq!(buf, b"hello");
495
496 // delete is idempotent.
497 store.delete(Namespace::Avatars, "u1").await.unwrap();
498 store.delete(Namespace::Avatars, "u1").await.unwrap();
499 assert!(!store.exists(Namespace::Avatars, "u1").await.unwrap());
500 }
501
502 #[tokio::test]
503 async fn put_file_consumes_source_and_shards_lfs_keys() {
504 let dir = tempfile::tempdir().unwrap();
505 let store = local_store(dir.path());
506 let src = dir.path().join("scratch.bin");
507 tokio::fs::write(&src, b"payload").await.unwrap();
508
509 // A sharded LFS key nests into subdirectories.
510 store
511 .put_file(
512 Namespace::Lfs,
513 "ab/cd/abcdef",
514 &src,
515 "application/octet-stream",
516 )
517 .await
518 .unwrap();
519 assert!(
520 !tokio::fs::try_exists(&src).await.unwrap(),
521 "source moved in"
522 );
523 assert_eq!(
524 store
525 .get_bytes(Namespace::Lfs, "ab/cd/abcdef")
526 .await
527 .unwrap()
528 .unwrap(),
529 b"payload"
530 );
531 assert!(dir.path().join("lfs/ab/cd/abcdef").exists());
532 }
533
534 #[test]
535 fn key_traversal_is_rejected() {
536 assert!(matches!(check_key(""), Err(BlobError::InvalidKey)));
537 assert!(matches!(
538 check_key("/etc/passwd"),
539 Err(BlobError::InvalidKey)
540 ));
541 assert!(matches!(check_key("a/../b"), Err(BlobError::InvalidKey)));
542 assert!(matches!(check_key("a//b"), Err(BlobError::InvalidKey)));
543 assert!(check_key("ab/cd/abcdef").is_ok());
544 assert!(check_key("01H0000000000000000000").is_ok());
545 }
546
547 #[tokio::test]
548 async fn missing_s3_config_is_an_error() {
549 let dir = tempfile::tempdir().unwrap();
550 let storage = Storage {
551 data_dir: dir.path().to_path_buf(),
552 repo_dir: dir.path().join("repos"),
553 lfs_dir: dir.path().join("lfs"),
554 backend: StorageBackend::S3,
555 s3: None,
556 };
557 assert!(matches!(
558 BlobStore::from_config(&storage),
559 Err(BlobError::MissingS3Config)
560 ));
561 }
562}
crates/config/src/lib.rs +65 −3
@@ -33,7 +33,8 @@ use figment::providers::{Env, Format, Serialized, Toml};
33pub use crate::secret::{REDACTED, Secret};33pub use crate::secret::{REDACTED, Secret};
34pub use crate::types::{34pub use crate::types::{
35 Auth, Captcha, CaptchaProvider, Config, Database, DefaultVisibility, Git, Instance, Log,35 Auth, Captcha, CaptchaProvider, Config, Database, DefaultVisibility, Git, Instance, Log,
36 LogFormat, LogLevel, Mail, MailBackend, MailEncryption, Search, Server, Ssh, Storage, Ui,36 LogFormat, LogLevel, Mail, MailBackend, MailEncryption, S3, Search, Server, Ssh, Storage,
37 StorageBackend, Ui,
37};38};
3839
39/// System-wide configuration path, the lowest-priority file source.40/// System-wide configuration path, the lowest-priority file source.
@@ -180,10 +181,33 @@ fn validate(config: &Config) -> Result<Vec<String>, ConfigError> {
180 errors.push("instance.default_branch must not be empty".to_string());181 errors.push("instance.default_branch must not be empty".to_string());
181 }182 }
182183
183 // Storage directories must exist or be creatable.184 // Storage directories must exist or be creatable. The bare repositories are
185 // always local; the LFS directory is only used by the local upload backend.
184 check_dir_creatable("storage.data_dir", &config.storage.data_dir, &mut errors);186 check_dir_creatable("storage.data_dir", &config.storage.data_dir, &mut errors);
185 check_dir_creatable("storage.repo_dir", &config.storage.repo_dir, &mut errors);187 check_dir_creatable("storage.repo_dir", &config.storage.repo_dir, &mut errors);
186 check_dir_creatable("storage.lfs_dir", &config.storage.lfs_dir, &mut errors);188 if config.storage.backend == StorageBackend::Local {
189 check_dir_creatable("storage.lfs_dir", &config.storage.lfs_dir, &mut errors);
190 }
191
192 // The S3 backend needs a bucket and credentials.
193 if config.storage.backend == StorageBackend::S3 {
194 match &config.storage.s3 {
195 None => errors.push(
196 "storage.backend is \"s3\" but the [storage.s3] section is missing".to_string(),
197 ),
198 Some(s3) => {
199 if s3.bucket.trim().is_empty() {
200 errors.push("storage.s3.bucket must not be empty".to_string());
201 }
202 if s3.access_key_id.trim().is_empty() {
203 errors.push("storage.s3.access_key_id must not be empty".to_string());
204 }
205 if s3.secret_access_key.expose().trim().is_empty() {
206 errors.push("storage.s3.secret_access_key must not be empty".to_string());
207 }
208 }
209 }
210 }
187211
188 // SMTP delivery needs a host and a from address.212 // SMTP delivery needs a host and a from address.
189 if config.mail.backend == MailBackend::Smtp {213 if config.mail.backend == MailBackend::Smtp {
@@ -338,6 +362,44 @@ mod tests {
338 }362 }
339363
340 #[test]364 #[test]
365 fn s3_backend_without_config_is_rejected() {
366 let err = from_toml("[storage]\nbackend = \"s3\"\n").unwrap_err();
367 match err {
368 ConfigError::Invalid(msg) => assert!(msg.contains("[storage.s3]"), "{msg}"),
369 other => panic!("expected Invalid, got {other:?}"),
370 }
371 }
372
373 #[test]
374 fn s3_backend_with_config_validates() {
375 let loaded = from_toml(
376 "[storage]\nbackend = \"s3\"\n\n[storage.s3]\nbucket = \"b\"\n\
377 access_key_id = \"k\"\nsecret_access_key = \"s\"\nendpoint = \"https://s3.example.com\"\n\
378 region = \"auto\"\npath_style = true\nprefix = \"fabrica/\"\n",
379 )
380 .unwrap();
381 let s3 = loaded.config.storage.s3.as_ref().unwrap();
382 assert_eq!(s3.bucket, "b");
383 assert_eq!(s3.region, "auto");
384 assert!(s3.path_style);
385 assert_eq!(s3.prefix, "fabrica/");
386 assert_eq!(s3.endpoint.as_deref(), Some("https://s3.example.com"));
387 }
388
389 #[test]
390 fn s3_backend_with_empty_bucket_is_rejected() {
391 let err = from_toml(
392 "[storage]\nbackend = \"s3\"\n\n[storage.s3]\nbucket = \"\"\n\
393 access_key_id = \"k\"\nsecret_access_key = \"s\"\n",
394 )
395 .unwrap_err();
396 match err {
397 ConfigError::Invalid(msg) => assert!(msg.contains("storage.s3.bucket"), "{msg}"),
398 other => panic!("expected Invalid, got {other:?}"),
399 }
400 }
401
402 #[test]
341 fn smtp_without_host_is_rejected() {403 fn smtp_without_host_is_rejected() {
342 let err = from_toml("[mail]\nbackend = \"smtp\"\n").unwrap_err();404 let err = from_toml("[mail]\nbackend = \"smtp\"\n").unwrap_err();
343 match err {405 match err {
crates/config/src/types.rs +56 −1
@@ -270,7 +270,11 @@ impl Default for Ssh {
270 }270 }
271}271}
272272
273/// `[storage]` — on-disk locations.273/// `[storage]` — on-disk locations and the blob backend for uploads.
274///
275/// Bare repositories always live on local disk (git needs filesystem access);
276/// only user uploads — avatars, release assets, and LFS objects — may be routed
277/// to S3-compatible object storage by setting `backend = "s3"`.
274#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]278#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
275#[serde(deny_unknown_fields)]279#[serde(deny_unknown_fields)]
276pub struct Storage {280pub struct Storage {
@@ -279,7 +283,14 @@ pub struct Storage {
279 /// Directory holding the bare repositories, sharded by id prefix.283 /// Directory holding the bare repositories, sharded by id prefix.
280 pub repo_dir: PathBuf,284 pub repo_dir: PathBuf,
281 /// Directory holding git-LFS objects, content-addressed by their sha256 oid.285 /// Directory holding git-LFS objects, content-addressed by their sha256 oid.
286 /// Used only by the `local` backend.
282 pub lfs_dir: PathBuf,287 pub lfs_dir: PathBuf,
288 /// Where uploads (avatars, release assets, LFS objects) are stored.
289 #[serde(default)]
290 pub backend: StorageBackend,
291 /// S3-compatible object storage settings, required when `backend = "s3"`.
292 #[serde(default)]
293 pub s3: Option<S3>,
283}294}
284295
285impl Default for Storage {296impl Default for Storage {
@@ -288,10 +299,54 @@ impl Default for Storage {
288 data_dir: PathBuf::from("/var/lib/fabrica"),299 data_dir: PathBuf::from("/var/lib/fabrica"),
289 repo_dir: PathBuf::from("/var/lib/fabrica/repos"),300 repo_dir: PathBuf::from("/var/lib/fabrica/repos"),
290 lfs_dir: PathBuf::from("/var/lib/fabrica/lfs"),301 lfs_dir: PathBuf::from("/var/lib/fabrica/lfs"),
302 backend: StorageBackend::Local,
303 s3: None,
291 }304 }
292 }305 }
293}306}
294307
308/// The upload storage backend.
309#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
310#[serde(rename_all = "lowercase")]
311pub enum StorageBackend {
312 /// Store uploads on the local filesystem (the default).
313 #[default]
314 Local,
315 /// Store uploads in an S3-compatible object store.
316 S3,
317}
318
319/// `[storage.s3]` — S3-compatible object storage. Works with AWS S3, `MinIO`,
320/// Cloudflare R2, Backblaze B2, and other implementations.
321#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
322#[serde(deny_unknown_fields)]
323pub struct S3 {
324 /// The bucket that holds all uploads.
325 pub bucket: String,
326 /// The region (e.g. `us-east-1`); many S3-compatible stores accept `auto`.
327 #[serde(default = "default_region")]
328 pub region: String,
329 /// A custom endpoint URL for non-AWS stores (`MinIO`, R2, …). Omit for AWS S3.
330 #[serde(default)]
331 pub endpoint: Option<String>,
332 /// The access key id.
333 pub access_key_id: String,
334 /// The secret access key.
335 pub secret_access_key: Secret,
336 /// Use path-style addressing (`endpoint/bucket/key`) rather than virtual-host
337 /// style. Required by most non-AWS stores such as `MinIO`.
338 #[serde(default)]
339 pub path_style: bool,
340 /// An optional key prefix prepended to every object (e.g. `fabrica/`).
341 #[serde(default)]
342 pub prefix: String,
343}
344
345/// The default S3 region when unspecified.
346fn default_region() -> String {
347 "us-east-1".to_string()
348}
349
295/// `[database]` — connection settings.350/// `[database]` — connection settings.
296#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]351#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
297#[serde(deny_unknown_fields)]352#[serde(deny_unknown_fields)]
docs/decisions.md +28 −0
@@ -667,3 +667,31 @@ optional label + pasted material), backed by `POST /settings/keys` and `/setting
667The add handler reuses `git::parse_public_key` for validation and fingerprinting and rejects a667The add handler reuses `git::parse_public_key` for validation and fingerprinting and rejects a
668duplicate fingerprint; delete is ownership-checked against `keys_by_user`. This closes the last668duplicate fingerprint; delete is ownership-checked against `keys_by_user`. This closes the last
669account-settings gap (§19) so key management no longer requires shell access.669account-settings gap (§19) so key management no longer requires shell access.
670
671## 2026-07-26 — Optional S3 object storage for uploads (aws-sdk-s3, aws-lc-rs)
672
673**Decision:** Add an optional S3-compatible backend for the three kinds of *uploads* — user
674avatars, release assets, and git-LFS objects — behind a new `blob` crate exposing a `BlobStore`
675enum with `Local` and `S3` variants. Bare git repositories are **not** included: git needs direct
676filesystem access, so `storage.repo_dir` is always local. Selected by `storage.backend =
677"local" | "s3"` (default `local`, so existing installs are unaffected) with a `[storage.s3]`
678section (`bucket`, `region`, `endpoint`, `access_key_id`, `secret_access_key`, `path_style`,
679`prefix`). The endpoint + path-style knobs make it work with MinIO, Cloudflare R2, Backblaze B2,
680and other S3-compatible stores, not just AWS.
681
682**Client:** `aws-sdk-s3` with a **manually constructed** config (static credentials, custom
683endpoint) rather than `aws-config`'s credential-provider chain, keeping the dependency surface and
684startup work small. The default HTTPS client (`default-https-client`) pulls `ring`, which the
685license gate forbids; instead the HTTP client is wired explicitly via `aws-smithy-http-client`
686with the `rustls-aws-lc` feature and `tls::Provider::Rustls(CryptoMode::AwsLc)`, so the crypto
687provider is **aws-lc-rs** — already in the tree via russh and already on the license allow-list —
688and `ring` never enters the graph (`cargo deny check licenses` stays green).
689
690**Streaming:** LFS and release-asset uploads still stream to a **local scratch file** under
691`data_dir/tmp` (LFS) / `data_dir/releases/tmp` (assets) so the sha256 (LFS) and size are computed
692before the object is servable; the verified temp file is then handed to `BlobStore::put_file`,
693which renames it into place (local) or uploads it and removes the temp (S3). Downloads stream
694straight from the backend (`open` returns size + an `AsyncRead`). Keys are namespaced
695(`avatars/`, `releases/`, `lfs/{shard}`) and validated against path traversal. The local backend
696preserves the historical on-disk layout (`data_dir/avatars`, `data_dir/releases`, `lfs_dir`), so
697no migration is needed to stay on local disk.
fabrica.example.toml +15 −1
@@ -41,7 +41,21 @@ clone_port = 22
41[storage]41[storage]
42data_dir = "/var/lib/fabrica" # secrets, host key, SQLite db, scratch42data_dir = "/var/lib/fabrica" # secrets, host key, SQLite db, scratch
43repo_dir = "/var/lib/fabrica/repos" # bare repos, sharded by id prefix43repo_dir = "/var/lib/fabrica/repos" # bare repos, sharded by id prefix
44lfs_dir = "/var/lib/fabrica/lfs" # git-LFS objects, content-addressed by sha25644lfs_dir = "/var/lib/fabrica/lfs" # git-LFS objects (local backend only)
45# Where uploads (avatars, release assets, LFS objects) live. Bare repos are
46# always local — only uploads may be offloaded to S3.
47backend = "local" # "local" (default) or "s3"
48
49# S3-compatible object storage (AWS S3, MinIO, Cloudflare R2, Backblaze B2, …).
50# Required when backend = "s3".
51# [storage.s3]
52# bucket = "fabrica"
53# region = "us-east-1" # many non-AWS stores accept "auto"
54# endpoint = "https://<account>.r2.cloudflarestorage.com" # omit for AWS
55# access_key_id = "..."
56# secret_access_key = "..."
57# path_style = false # true for MinIO and most non-AWS stores
58# prefix = "" # optional key prefix, e.g. "fabrica/"
4559
46[database]60[database]
47url = "sqlite:///var/lib/fabrica/fabrica.db"61url = "sqlite:///var/lib/fabrica/fabrica.db"