feat: synapse integration

Author: rvagg

.gitignore

@@ -2,3 +2,9 @@ node_modules/
 boxo/
 kubo/
 dist/
+.vite/
+*.car
+test-output/
+test-car-output/
+test-e2e-cars/
+test-simple-car-output/

README.md

@@ -6,6 +6,18 @@ An IPFS Pinning Service API implementation that pins to Filecoin's PDP service,
 
 Filecoin Pin is a TypeScript daemon that implements the [IPFS Pinning Service API](https://ipfs.github.io/pinning-services-api-spec/) to enable users to pin IPFS content to Filecoin using familiar IPFS tooling like Kubo's `ipfs pin remote` commands.
 
+### How It Works
+
+1. **Serve Pin Service**: The daemon runs an HTTP server that implements the IPFS Pinning Service API
+2. **Receive Pin Requests**: When you run `ipfs pin remote add`, Kubo sends a pin request to the service
+3. **Fetch Blocks from IPFS**: The service connects to the IPFS network and fetches blocks for the requested CID (usually from the requesting node itself)
+4. **Store in CAR**: As blocks arrive, they're written directly to a CAR (Content Addressable aRchive) file on disk
+5. **Upload to PDP Provider**: Once all blocks are collected, the CAR file is uploaded to a Proof of Data Possession (PDP) service provider
+6. **Commit to Filecoin**: The PDP provider commits the data to the Filecoin blockchain
+7. **Start Proving**: The storage provider begins generating ongoing proofs that they still possess your data
+
+This bridges the gap between IPFS's content-addressed storage and Filecoin's incentivized persistence layer, giving you the best of both worlds - easy pinning with long-term storage guarantees.
+
 **⚠️ Alpha Software**: This is currently alpha software, only deploying on Filecoin's Calibration Test network with storage providers participating in network testing, not dedicating long-term persistence.
 
 You need a Filecoin calibration network wallet funded with USDFC. See the [USDFC documentation](https://docs.secured.finance/usdfc-stablecoin/getting-started) which has a "Testnet Resources" section for getting USDFC on calibnet.
@@ -48,8 +60,8 @@ Configuration is managed through environment variables. The service uses platfor
 #### Environment Variables
 
 ```bash
-# Required for Filecoin operations
-export PRIVATE_KEY="your-filecoin-private-key"      # Ethereum private key (must be funded with USDFC)
+# REQUIRED - Without this, the service will not start
+export PRIVATE_KEY="your-filecoin-private-key"      # Ethereum private key (must be funded with USDFC on calibration network)
 
 # Optional configuration with defaults
 export PORT=3456                                    # API server port (default: 3456)
@@ -71,12 +83,14 @@ When `DATABASE_PATH` and `CAR_STORAGE_PATH` are not specified, the service uses
 
 ### Running the Daemon
 
+⚠️ **PRIVATE_KEY is required** - The service will not start without it.
+
 ```bash
 # Start the daemon
-npm start
+PRIVATE_KEY=0x... npm start
 
 # Or with custom configuration
-PORT=8080 npm start
+PRIVATE_KEY=0x... PORT=8080 RPC_URL=wss://... npm start
 ```
 
 ### CLI Usage
@@ -115,9 +129,11 @@ ipfs pin remote ls --service=filecoin-pin --status=pinning,queued,pinned
 - `npm run build` - Compile TypeScript to JavaScript
 - `npm run dev` - Start development server with hot reload
 - `npm start` - Run compiled output
-- `npm test` - Run tests with type checking
+- `npm test` - Run linting, type checking, unit tests, and integration tests
+- `npm run test:unit` - Run unit tests only
+- `npm run test:integration` - Run integration tests only
 - `npm run test:watch` - Run tests in watch mode
-- `npm run lint` - Check code style
+- `npm run lint` - Check code style with ts-standard
 - `npm run lint:fix` - Auto-fix code style issues
 - `npm run typecheck` - Type check without emitting files
 

package.json

@@ -8,17 +8,31 @@
   "bin": {
     "filecoin-pin": "dist/cli.js"
   },
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "dist/**/*.js.map",
+    "dist/**/*.d.ts.map",
+    "src/**/*.ts",
+    "README.md",
+    "LICENSE.md"
+  ],
   "scripts": {
     "build": "tsc",
-    "dev": "tsx watch src/index.ts",
-    "start": "node dist/index.js",
-    "test": "npm run typecheck && npm run test:unit && npm run test:e2e",
-    "test:unit": "vitest run test/unit",
-    "test:e2e": "vitest run test/e2e",
+    "dev": "tsx watch src/cli.ts daemon",
+    "start": "node dist/cli.js daemon",
+    "test": "npm run lint && npm run typecheck && npm run test:unit && npm run test:integration",
+    "test:unit": "vitest run src/test/unit",
+    "test:integration": "vitest run src/test/integration",
     "test:watch": "vitest",
     "lint": "ts-standard 'src/**/*.ts'",
     "lint:fix": "ts-standard 'src/**/*.ts' --fix",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "npm run build"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/FilOzone/filecoin-pin.git"
   },
   "keywords": [
     "ipfs",
@@ -29,10 +43,12 @@
   ],
   "author": "Rod Vagg <[email protected]>",
   "license": "Apache-2.0 OR MIT",
-  "engines": {
-    "node": ">=24.0.0"
+  "bugs": {
+    "url": "https://github.com/FilOzone/filecoin-pin/issues"
   },
+  "homepage": "https://github.com/FilOzone/filecoin-pin#readme",
   "dependencies": {
+    "@filoz/synapse-sdk": "^0.20.0",
     "@ipld/car": "^5.4.2",
     "fastify": "^5.4.0",
     "helia": "^5.5.0",
@@ -47,5 +63,8 @@
     "tsx": "^4.20.3",
     "typescript": "^5.8.3",
     "vitest": "^3.2.4"
+  },
+  "publishConfig": {
+    "access": "public"
   }
 }

src/cli.ts

@@ -23,17 +23,18 @@ Usage:
   filecoin-pin help      Show this help message
 
 Environment Variables:
-  PORT                   API server port (default: 3456)
-  HOST                   API server host (default: localhost)
-  PRIVATE_KEY           Ethereum private key for Filecoin transactions
+  PRIVATE_KEY           Private key for Filecoin transactions (required)
+  PORT                  API server port (default: 3456)
+  HOST                  API server host (default: localhost)
   RPC_URL               Filecoin RPC endpoint (default: calibration testnet)
-  DATABASE_PATH         SQLite database location (default: ./pins.db)
-  CAR_STORAGE_PATH      Temporary CAR file directory (default: ./cars)
+  DATABASE_PATH         SQLite database location (default: {config}/pins.db)
+  CAR_STORAGE_PATH      Temporary CAR file directory (default: {config}/cars)
   LOG_LEVEL             Log level (default: info)
+  PANDORA_ADDRESS       Override Pandora contract address (optional)
 
-Example:
-  filecoin-pin daemon
-  PORT=8080 PRIVATE_KEY=0x... filecoin-pin daemon
+Examples:
+  PRIVATE_KEY=0x... filecoin-pin daemon
+  PORT=8080 PRIVATE_KEY=0x... RPC_URL=wss://... filecoin-pin daemon
 `)
 }
 

src/config.ts

@@ -9,6 +9,7 @@ export interface Config {
   databasePath: string
   carStoragePath: string
   logLevel: string
+  pandoraAddress: string | undefined
 }
 
 function getDataDirectory (): string {
@@ -44,6 +45,7 @@ export function createConfig (): Config {
     rpcUrl: process.env.RPC_URL ?? 'https://api.calibration.node.glif.io/rpc/v1',
     databasePath: process.env.DATABASE_PATH ?? join(dataDir, 'pins.db'),
     carStoragePath: process.env.CAR_STORAGE_PATH ?? join(dataDir, 'cars'),
-    logLevel: process.env.LOG_LEVEL ?? 'info'
+    logLevel: process.env.LOG_LEVEL ?? 'info',
+    pandoraAddress: process.env.PANDORA_ADDRESS
   }
 }

src/filecoin-pin-store.ts

@@ -1,4 +1,4 @@
-import { unlink } from 'node:fs/promises'
+import { unlink, readFile } from 'node:fs/promises'
 import { EventEmitter } from 'node:events'
 import { join } from 'node:path'
 import { createPinningHeliaNode } from './create-pinning-helia.js'
@@ -7,6 +7,7 @@ import type { Config } from './config.js'
 import type { Helia } from 'helia'
 import type { CID } from 'multiformats/cid'
 import type { Logger } from 'pino'
+import type { SynapseService } from './synapse-service.js'
 
 export interface PinningServiceUser {
   id: string
@@ -37,6 +38,9 @@ export interface FilecoinPinMetadata {
   carStats: CARBlockstoreStats
   pinStarted: number
   pinCompleted?: number
+  synapseCommP?: string
+  synapseRootId?: number
+  synapseProofSetId?: string
 }
 
 export interface FilecoinStoredPinStatus extends StoredPinStatus {
@@ -46,6 +50,7 @@ export interface FilecoinStoredPinStatus extends StoredPinStatus {
 export interface FilecoinPinStoreInit {
   config: Config
   logger: Logger
+  synapseService: SynapseService
 }
 
 /**
@@ -54,6 +59,7 @@ export interface FilecoinPinStoreInit {
 export class FilecoinPinStore extends EventEmitter {
   private readonly config: Config
   private readonly logger: Logger
+  private readonly synapseService: SynapseService
   private readonly pins = new Map<string, FilecoinStoredPinStatus>()
   private readonly activePins = new Map<string, {
     helia: Helia
@@ -67,6 +73,7 @@ export class FilecoinPinStore extends EventEmitter {
     super()
     this.config = init.config
     this.logger = init.logger
+    this.synapseService = init.synapseService
   }
 
   async start (): Promise<void> {
@@ -276,6 +283,86 @@ export class FilecoinPinStore extends EventEmitter {
         missingBlocks: finalStats.missingBlocks.size
       }, 'CAR file finalized')
 
+      // Store on Filecoin
+      try {
+        // Read the CAR file (streaming not yet supported in Synapse)
+        const carData = await readFile(pinStatus.filecoin.carFilePath)
+
+        // Upload using Synapse
+        const synapseResult = await this.synapseService.storage.upload(carData, {
+          onUploadComplete: (commp) => {
+            this.logger.info({
+              event: 'synapse.upload.piece_uploaded',
+              pinId,
+              commp: commp.toString()
+            }, 'Upload to PDP server complete')
+          },
+          onRootAdded: (transaction) => {
+            if (transaction != null) {
+              this.logger.info({
+                event: 'synapse.upload.root_added',
+                pinId,
+                txHash: transaction.hash
+              }, 'Root addition transaction submitted')
+            } else {
+              this.logger.info({
+                event: 'synapse.upload.root_added',
+                pinId
+              }, 'Root added to proof set')
+            }
+          },
+          onRootConfirmed: (rootIds) => {
+            this.logger.info({
+              event: 'synapse.upload.root_confirmed',
+              pinId,
+              rootIds
+            }, 'Root addition confirmed on-chain')
+          }
+        })
+
+        // Store Synapse metadata
+        pinStatus.filecoin.synapseCommP = synapseResult.commp.toString()
+        if (synapseResult.rootId !== undefined) {
+          pinStatus.filecoin.synapseRootId = synapseResult.rootId
+        }
+        pinStatus.filecoin.synapseProofSetId = this.synapseService.storage.proofSetId
+
+        // Add to info for API response
+        pinStatus.info = {
+          ...pinStatus.info,
+          synapse_commp: synapseResult.commp.toString(),
+          synapse_root_id: (synapseResult.rootId ?? 0).toString(),
+          synapse_proof_set_id: this.synapseService.storage.proofSetId
+        }
+
+        this.logger.info({
+          event: 'synapse.upload.success',
+          pinId,
+          commp: synapseResult.commp,
+          rootId: synapseResult.rootId,
+          proofSetId: this.synapseService.storage.proofSetId
+        }, 'Successfully uploaded to Filecoin with Synapse')
+      } catch (error) {
+        // Rollback on Synapse failure
+        this.logger.error({
+          event: 'synapse.upload.failed',
+          pinId,
+          error
+        }, 'Failed to upload to Filecoin with Synapse, rolling back')
+
+        // Clean up the CAR file
+        try {
+          await blockstore.cleanup()
+          await unlink(pinStatus.filecoin.carFilePath)
+          this.logger.info({ pinId, carFilePath: pinStatus.filecoin.carFilePath }, 'Deleted CAR file after Synapse failure')
+        } catch (cleanupError) {
+          this.logger.warn({ pinId, error: cleanupError }, 'Failed to clean up CAR file')
+        }
+
+        // Re-throw to mark pin as failed
+        throw new Error(`Synapse upload failed: ${error instanceof Error ? error.message : 'Unknown error'}`)
+      }
+
       // Update pin status to completed
       pinStatus.status = 'pinned'
       pinStatus.filecoin.carStats = finalStats
@@ -299,13 +386,6 @@ export class FilecoinPinStore extends EventEmitter {
         carFilePath: pinStatus.filecoin.carFilePath
       })
 
-      // CAR file is ready for upload to Filecoin when needed
-      this.logger.info({
-        pinId,
-        cid: cid.toString(),
-        carFilePath: pinStatus.filecoin.carFilePath
-      }, 'CAR file ready for Filecoin upload')
-
       this.logger.info({ pinId, cid: cid.toString() }, 'Pin processing completed successfully')
     } catch (error) {
       this.logger.error({ pinId, cid: cid.toString(), error }, 'Pin processing failed')

src/filecoin-pinning-server.ts

@@ -4,6 +4,7 @@ import type { Config } from './config.js'
 import type { Logger } from 'pino'
 import { CID } from 'multiformats/cid'
 import type { ServiceInfo } from './index.js'
+import { initializeSynapse } from './synapse-service.js'
 
 declare module 'fastify' {
   interface FastifyRequest {
@@ -19,11 +20,15 @@ const DEFAULT_USER_INFO = {
   name: 'Default User'
 }
 
-export async function createFilecoinPinningServer (config: Config, logger: Logger, serviceInfo: ServiceInfo): Promise<any> {
-  // Create our custom Filecoin pin store
+export async function createFilecoinPinningServer (config: Config, logger: Logger, serviceInfo: ServiceInfo): Promise<{ server: FastifyInstance, pinStore: FilecoinPinStore }> {
+  // Initialize Synapse
+  const synapseService = await initializeSynapse(config, logger)
+
+  // Create our custom Filecoin pin store with Synapse service
   const filecoinPinStore = new FilecoinPinStore({
     config,
-    logger
+    logger,
+    synapseService
   })
 
   // Set up event handlers for monitoring
@@ -122,10 +127,7 @@ export async function createFilecoinPinningServer (config: Config, logger: Logge
   logger.info('Filecoin pinning service API server started')
 
   return {
-    server: {
-      server: server.server, // Expose underlying HTTP server for address()
-      close: async () => await server.close()
-    },
+    server,
     pinStore: filecoinPinStore
   }
 }