fix!: treat validity as opaque (ipfs#307)

Instead of parsing the validity field of an IPNS record as a timestamp
during unmarshalling, treat it as an opaque string value.

This ensures we can round-trip records supplied by other systems which
may have a different interpretation of rfc3339 dates, for example.

It also means we can handle different types of validity types, not
just EOL.

BREAKING CHANGE: the validity field is now a string

src/index.ts

@@ -39,7 +39,7 @@ export interface IPNSRecordV1V2 {
   /**
    * expiration datetime for the record in RFC3339 format
    */
-  validity: NanoDate
+  validity: string
 
   /**
    * number representing the version of the record
@@ -84,9 +84,10 @@ export interface IPNSRecordV2 {
   validityType: IpnsEntry.ValidityType
 
   /**
-   * expiration datetime for the record in RFC3339 format
+   * If the validity type is EOL, this is the expiration datetime for the record
+   * in RFC3339 format
    */
-  validity: NanoDate
+  validity: string
 
   /**
    * number representing the version of the record
@@ -169,7 +170,7 @@ export async function create (peerId: PeerId, value: CID | PeerId | string, seq:
   const [ms, ns] = lifetime.toString().split('.')
   const lifetimeNs = (BigInt(ms) * BigInt(100000)) + BigInt(ns ?? '0')
 
-  return _create(peerId, value, seq, validityType, expirationDate, lifetimeNs, options)
+  return _create(peerId, value, seq, validityType, expirationDate.toString(), lifetimeNs, options)
 }
 
 /**
@@ -198,12 +199,12 @@ export async function createWithExpiration (peerId: PeerId, value: CID | PeerId
   const ttlMs = expirationDate.toDate().getTime() - Date.now()
   const ttlNs = (BigInt(ttlMs) * BigInt(100000)) + BigInt(expirationDate.getNano())
 
-  return _create(peerId, value, seq, validityType, expirationDate, ttlNs, options)
+  return _create(peerId, value, seq, validityType, expirationDate.toString(), ttlNs, options)
 }
 
-const _create = async (peerId: PeerId, value: CID | PeerId | string, seq: number | bigint, validityType: IpnsEntry.ValidityType, expirationDate: NanoDate, ttl: bigint, options: CreateOptions = defaultCreateOptions): Promise<IPNSRecord> => {
+const _create = async (peerId: PeerId, value: CID | PeerId | string, seq: number | bigint, validityType: IpnsEntry.ValidityType, validity: string, ttl: bigint, options: CreateOptions = defaultCreateOptions): Promise<IPNSRecord> => {
   seq = BigInt(seq)
-  const isoValidity = uint8ArrayFromString(expirationDate.toString())
+  const isoValidity = uint8ArrayFromString(validity)
   const normalizedValue = normalizeValue(value)
   const encodedValue = uint8ArrayFromString(normalizedValue)
 
@@ -212,7 +213,7 @@ const _create = async (peerId: PeerId, value: CID | PeerId | string, seq: number
   }
 
   const privateKey = await unmarshalPrivateKey(peerId.privateKey)
-  const data = createCborData(encodedValue, isoValidity, validityType, seq, ttl)
+  const data = createCborData(encodedValue, validityType, isoValidity, seq, ttl)
   const sigData = ipnsRecordDataForV2Sig(data)
   const signatureV2 = await privateKey.sign(sigData)
   let pubKey: Uint8Array | undefined
@@ -233,7 +234,7 @@ const _create = async (peerId: PeerId, value: CID | PeerId | string, seq: number
     const record: IPNSRecord = {
       value: normalizedValue,
       signatureV1,
-      validity: expirationDate,
+      validity,
       validityType,
       sequence: seq,
       ttl,
@@ -249,7 +250,7 @@ const _create = async (peerId: PeerId, value: CID | PeerId | string, seq: number
   } else {
     const record: IPNSRecordV2 = {
       value: normalizedValue,
-      validity: expirationDate,
+      validity,
       validityType,
       sequence: seq,
       ttl,

src/selector.ts

@@ -1,3 +1,5 @@
+import NanoDate from 'timestamp-nano'
+import { IpnsEntry } from './pb/ipns.js'
 import { unmarshal } from './utils.js'
 
 export function ipnsSelector (key: Uint8Array, data: Uint8Array[]): number {
@@ -21,16 +23,18 @@ export function ipnsSelector (key: Uint8Array, data: Uint8Array[]): number {
       return 1
     }
 
-    // choose longer lived record if sequence numbers the same
-    const recordAValidityDate = a.record.validity.toDate()
-    const recordBValidityDate = b.record.validity.toDate()
+    if (a.record.validityType === IpnsEntry.ValidityType.EOL && b.record.validityType === IpnsEntry.ValidityType.EOL) {
+      // choose longer lived record if sequence numbers the same
+      const recordAValidityDate = NanoDate.fromString(a.record.validity).toDate()
+      const recordBValidityDate = NanoDate.fromString(b.record.validity).toDate()
 
-    if (recordAValidityDate.getTime() > recordBValidityDate.getTime()) {
-      return -1
-    }
+      if (recordAValidityDate.getTime() > recordBValidityDate.getTime()) {
+        return -1
+      }
 
-    if (recordAValidityDate.getTime() < recordBValidityDate.getTime()) {
-      return 1
+      if (recordAValidityDate.getTime() < recordBValidityDate.getTime()) {
+        return 1
+      }
     }
 
     return 0

src/utils.ts

@@ -6,7 +6,6 @@ import * as cborg from 'cborg'
 import errCode from 'err-code'
 import { base36 } from 'multiformats/bases/base36'
 import { CID } from 'multiformats/cid'
-import NanoDate from 'timestamp-nano'
 import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
 import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
@@ -20,53 +19,6 @@ const log = logger('ipns:utils')
 const IPNS_PREFIX = uint8ArrayFromString('/ipns/')
 const LIBP2P_CID_CODEC = 114
 
-/**
- * Convert a JavaScript date into an `RFC3339Nano` formatted
- * string
- */
-export function toRFC3339 (time: Date): string {
-  const year = time.getUTCFullYear()
-  const month = String(time.getUTCMonth() + 1).padStart(2, '0')
-  const day = String(time.getUTCDate()).padStart(2, '0')
-  const hour = String(time.getUTCHours()).padStart(2, '0')
-  const minute = String(time.getUTCMinutes()).padStart(2, '0')
-  const seconds = String(time.getUTCSeconds()).padStart(2, '0')
-  const milliseconds = time.getUTCMilliseconds()
-  const nanoseconds = milliseconds * 1000 * 1000
-
-  return `${year}-${month}-${day}T${hour}:${minute}:${seconds}.${nanoseconds}Z`
-}
-
-/**
- * Parses a date string formatted as `RFC3339Nano` into a
- * JavaScript Date object
- */
-export function parseRFC3339 (time: string): Date {
-  const rfc3339Matcher = new RegExp(
-    // 2006-01-02T
-    '(\\d{4})-(\\d{2})-(\\d{2})T' +
-    // 15:04:05
-    '(\\d{2}):(\\d{2}):(\\d{2})' +
-    // .999999999Z
-    '\\.(\\d+)Z'
-  )
-  const m = String(time).trim().match(rfc3339Matcher)
-
-  if (m == null) {
-    throw new Error('Invalid format')
-  }
-
-  const year = parseInt(m[1], 10)
-  const month = parseInt(m[2], 10) - 1
-  const date = parseInt(m[3], 10)
-  const hour = parseInt(m[4], 10)
-  const minute = parseInt(m[5], 10)
-  const second = parseInt(m[6], 10)
-  const millisecond = parseInt(m[7].padEnd(6, '0').slice(0, 3), 10)
-
-  return new Date(Date.UTC(year, month, date, hour, minute, second, millisecond))
-}
-
 /**
  * Extracts a public key from the passed PeerId, falling
  * back to the pubKey embedded in the ipns record
@@ -129,7 +81,7 @@ export const marshal = (obj: IPNSRecord | IPNSRecordV2): Uint8Array => {
       value: uint8ArrayFromString(obj.value),
       signatureV1: obj.signatureV1,
       validityType: obj.validityType,
-      validity: uint8ArrayFromString(obj.validity.toString()),
+      validity: uint8ArrayFromString(obj.validity),
       sequence: obj.sequence,
       ttl: obj.ttl,
       pubKey: obj.pubKey,
@@ -167,14 +119,7 @@ export function unmarshal (buf: Uint8Array): IPNSRecord {
 
   const data = parseCborData(message.data)
   const value = normalizeValue(data.Value)
-
-  let validity
-  try {
-    validity = NanoDate.fromDate(parseRFC3339(uint8ArrayToString(data.Validity)))
-  } catch (e) {
-    log.error('unrecognized validity format (not an rfc3339 format)')
-    throw errCode(new Error('unrecognized validity format (not an rfc3339 format)'), ERRORS.ERR_UNRECOGNIZED_FORMAT)
-  }
+  const validity = uint8ArrayToString(data.Validity)
 
   if (message.value != null && message.signatureV1 != null) {
     // V1+V2
@@ -219,7 +164,7 @@ export const peerIdFromRoutingKey = (key: Uint8Array): PeerId => {
   return peerIdFromBytes(key.slice(IPNS_PREFIX.length))
 }
 
-export const createCborData = (value: Uint8Array, validity: Uint8Array, validityType: string, sequence: bigint, ttl: bigint): Uint8Array => {
+export const createCborData = (value: Uint8Array, validityType: IpnsEntry.ValidityType, validity: Uint8Array, sequence: bigint, ttl: bigint): Uint8Array => {
   let ValidityType
 
   if (validityType === IpnsEntry.ValidityType.EOL) {

src/validator.ts

@@ -1,5 +1,6 @@
 import { logger } from '@libp2p/logger'
 import errCode from 'err-code'
+import NanoDate from 'timestamp-nano'
 import * as ERRORS from './errors.js'
 import { IpnsEntry } from './pb/ipns.js'
 import { extractPublicKey, ipnsRecordDataForV2Sig, unmarshal, peerIdFromRoutingKey } from './utils.js'
@@ -37,7 +38,7 @@ export const validate = async (publicKey: PublicKey, buf: Uint8Array): Promise<v
 
   // Validate according to the validity type
   if (record.validityType === IpnsEntry.ValidityType.EOL) {
-    if (record.validity.toDate().getTime() < Date.now()) {
+    if (NanoDate.fromString(record.validity).toDate().getTime() < Date.now()) {
       log.error('record has expired')
       throw errCode(new Error('record has expired'), ERRORS.ERR_IPNS_EXPIRED_RECORD)
     }

test/conformance.spec.ts

@@ -65,4 +65,21 @@ describe('conformance', function () {
 
     expect(record.value).to.equal('/ipfs/bafkqadtwgiww63tmpeqhezldn5zgi')
   })
+
+  it('should round trip fixtures', () => {
+    const fixtures = [
+      'test/fixtures/k51qzi5uqu5dlkw8pxuw9qmqayfdeh4kfebhmreauqdc6a7c3y7d5i9fi8mk9w_v1-v2.ipns-record',
+      'test/fixtures/k51qzi5uqu5diamp7qnnvs1p1gzmku3eijkeijs3418j23j077zrkok63xdm8c_v1-v2-broken-signature-v2.ipns-record',
+      'test/fixtures/k51qzi5uqu5dilgf7gorsh9vcqqq4myo6jd4zmqkuy9pxyxi5fua3uf7axph4y_v1-v2-broken-signature-v1.ipns-record',
+      'test/fixtures/k51qzi5uqu5dit2ku9mutlfgwyz8u730on38kd10m97m36bjt66my99hb6103f_v2.ipns-record'
+    ]
+
+    for (const fixture of fixtures) {
+      const buf = loadFixture(fixture)
+      const record = ipns.unmarshal(buf)
+      const marshalled = ipns.marshal(record)
+
+      expect(buf).to.equalBytes(marshalled)
+    }
+  })
 })

test/fixtures/records.ts

@@ -0,0 +1,7 @@
+import { peerIdFromString } from '@libp2p/peer-id'
+
+export const kuboRecord = {
+  bytes: Uint8Array.from([10, 52, 47, 105, 112, 102, 115, 47, 81, 109, 97, 52, 115, 87, 121, 111, 84, 105, 74, 75, 89, 120, 49, 119, 84, 106, 107, 120, 87, 89, 55, 49, 100, 89, 103, 49, 111, 87, 103, 69, 55, 83, 69, 57, 111, 105, 84, 71, 113, 71, 110, 121, 111, 82, 18, 64, 178, 225, 212, 157, 188, 23, 25, 166, 9, 89, 255, 63, 227, 160, 140, 70, 192, 237, 178, 167, 94, 6, 112, 184, 106, 130, 89, 252, 141, 158, 84, 53, 65, 125, 253, 93, 255, 17, 28, 93, 9, 176, 232, 89, 51, 118, 104, 236, 126, 137, 136, 72, 0, 127, 101, 88, 178, 83, 115, 6, 30, 28, 140, 5, 24, 0, 34, 27, 50, 48, 50, 52, 45, 48, 49, 45, 49, 57, 84, 49, 54, 58, 51, 51, 58, 50, 48, 46, 56, 53, 56, 50, 48, 57, 90, 40, 0, 48, 128, 192, 226, 133, 227, 104, 66, 64, 133, 91, 52, 64, 253, 186, 129, 154, 218, 85, 188, 18, 104, 96, 180, 216, 254, 176, 210, 145, 130, 209, 176, 150, 134, 33, 59, 197, 162, 193, 15, 252, 71, 190, 240, 25, 3, 169, 60, 24, 236, 68, 218, 171, 61, 235, 157, 73, 215, 0, 51, 52, 24, 195, 90, 158, 245, 199, 172, 204, 12, 249, 89, 7, 74, 136, 1, 165, 99, 84, 84, 76, 27, 0, 0, 3, 70, 48, 184, 160, 0, 101, 86, 97, 108, 117, 101, 88, 52, 47, 105, 112, 102, 115, 47, 81, 109, 97, 52, 115, 87, 121, 111, 84, 105, 74, 75, 89, 120, 49, 119, 84, 106, 107, 120, 87, 89, 55, 49, 100, 89, 103, 49, 111, 87, 103, 69, 55, 83, 69, 57, 111, 105, 84, 71, 113, 71, 110, 121, 111, 82, 104, 83, 101, 113, 117, 101, 110, 99, 101, 0, 104, 86, 97, 108, 105, 100, 105, 116, 121, 88, 27, 50, 48, 50, 52, 45, 48, 49, 45, 49, 57, 84, 49, 54, 58, 51, 51, 58, 50, 48, 46, 56, 53, 56, 50, 48, 57, 90, 108, 86, 97, 108, 105, 100, 105, 116, 121, 84, 121, 112, 101, 0]),
+  peerId: peerIdFromString('12D3KooWBT21CjaZgY3MvoFFwRJLBEqgk7zwa294Boh9wdX2RUX2'),
+  value: '/ipfs/Qma4sWyoTiJKYx1wTjkxWY71dYg1oWgE7SE9oiTGqGnyoR'
+}

test/index.spec.ts

@@ -16,6 +16,7 @@ import * as ipns from '../src/index.js'
 import { IpnsEntry } from '../src/pb/ipns.js'
 import { extractPublicKey, peerIdToRoutingKey, parseCborData, createCborData, ipnsRecordDataForV2Sig } from '../src/utils.js'
 import { ipnsValidator } from '../src/validator.js'
+import { kuboRecord } from './fixtures/records.js'
 import type { PeerId } from '@libp2p/interface'
 
 describe('ipns', function () {
@@ -186,7 +187,7 @@ describe('ipns', function () {
     const record = await ipns.create(peerId, inputValue, 0, 1000000)
 
     const pb = IpnsEntry.decode(ipns.marshal(record))
-    pb.data = createCborData(uint8ArrayFromString(inputValue), pb.validity ?? new Uint8Array(0), pb.validityType ?? '', pb.sequence ?? 0n, pb.ttl ?? 0n)
+    pb.data = createCborData(uint8ArrayFromString(inputValue), pb.validityType ?? IpnsEntry.ValidityType.EOL, pb.validity ?? new Uint8Array(0), pb.sequence ?? 0n, pb.ttl ?? 0n)
     pb.value = uint8ArrayFromString(inputValue)
 
     const modifiedRecord = ipns.unmarshal(IpnsEntry.encode(pb))
@@ -199,7 +200,7 @@ describe('ipns', function () {
     const record = await ipns.create(peerId, inputValue, 0, 1000000)
 
     const pb = IpnsEntry.decode(ipns.marshal(record))
-    pb.data = createCborData(uint8ArrayFromString(inputValue), pb.validity ?? new Uint8Array(0), pb.validityType ?? '', pb.sequence ?? 0n, pb.ttl ?? 0n)
+    pb.data = createCborData(uint8ArrayFromString(inputValue), pb.validityType ?? IpnsEntry.ValidityType.EOL, pb.validity ?? new Uint8Array(0), pb.sequence ?? 0n, pb.ttl ?? 0n)
     pb.value = uint8ArrayFromString(inputValue)
 
     const modifiedRecord = ipns.unmarshal(IpnsEntry.encode(pb))
@@ -397,4 +398,19 @@ describe('ipns', function () {
 
     expect(record).to.have.property('value', '/ipfs/bafkqae3imvwgy3zamzzg63janjzs22lqnzzqu')
   })
+
+  it('should round trip kubo records to bytes and back', async () => {
+    // the IPNS spec gives an example for the Validity field as
+    // 1970-01-01T00:00:00.000000001Z - e.g. nanosecond precision but Kubo only
+    // uses microsecond precision. The value is a timestamp as defined by
+    // rfc3339 which doesn't have a strong opinion on fractions of seconds so
+    // both are valid but we must be able to round trip them intact.
+    const unmarshalled = ipns.unmarshal(kuboRecord.bytes)
+    const remarhshalled = ipns.marshal(unmarshalled)
+
+    const reUnmarshalled = ipns.unmarshal(remarhshalled)
+
+    expect(unmarshalled).to.deep.equal(reUnmarshalled)
+    expect(remarhshalled).to.equalBytes(kuboRecord.bytes)
+  })
 })