Print a JSON representation of part of the header [#63] (#159)

Print a JSON representation of part of the header [#63]
* refactor show to take io.Writer instead of os.Stdout
* add integration test for Show command
* hide write command as it's not implemented yet.

.github/workflows/test.yml

@@ -27,6 +27,6 @@ jobs:
       - run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then exit 1; fi
       - run: go vet caddy/pmtiles_proxy.go
       - run: go vet main.go
-      - run: go vet pmtiles/*
+      - run: go vet ./pmtiles
       - name: Run Revive Action by pulling pre-built image
         uses: docker://morphy/revive-action:v2

.gitignore

@@ -1,5 +1,7 @@
 dist/
 go-pmtiles
-*.pmtiles
+/*.pmtiles
+/*.mbtiles
+*.json
 *.geojson
 *.tsv.gz

main.go

@@ -36,11 +36,12 @@ var cli struct {
 	} `cmd:"" help:"Convert an MBTiles or older spec version to PMTiles."`
 
 	Show struct {
-		Path      string `arg:""`
-		Bucket    string `help:"Remote bucket"`
-		Metadata  bool   `help:"Print only the JSON metadata."`
-		Tilejson  bool   `help:"Print the TileJSON."`
-		PublicURL string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles"`
+		Path       string `arg:""`
+		Bucket     string `help:"Remote bucket"`
+		Metadata   bool   `help:"Print only the JSON metadata."`
+		HeaderJson bool   `help:"Print a JSON representation of the header information."`
+		Tilejson   bool   `help:"Print the TileJSON."`
+		PublicURL  string `help:"Public base URL of tile endpoint for TileJSON e.g. https://example.com/tiles"`
 	} `cmd:"" help:"Inspect a local or remote archive."`
 
 	Tile struct {
@@ -51,6 +52,12 @@ var cli struct {
 		Bucket string `help:"Remote bucket"`
 	} `cmd:"" help:"Fetch one tile from a local or remote archive and output on stdout."`
 
+	Write struct {
+		Input      string `arg:"" help:"Input archive file." type:"existingfile"`
+		HeaderJson string `help:"Input header JSON file (written by show --header-json)." type:"existingfile"`
+		Metadata   string `help:"Input metadata JSON (written by show --metadata)." type:"existingfile"`
+	} `cmd:"" help:"Write header data or metadata to an existing archive." hidden:""`
+
 	Extract struct {
 		Input           string  `arg:"" help:"Input local or remote archive."`
 		Output          string  `arg:"" help:"Output archive." type:"path"`
@@ -122,12 +129,12 @@ func main() {
 
 	switch ctx.Command() {
 	case "show <path>":
-		err := pmtiles.Show(logger, cli.Show.Bucket, cli.Show.Path, cli.Show.Metadata, cli.Show.Tilejson, cli.Show.PublicURL, false, 0, 0, 0)
+		err := pmtiles.Show(logger, os.Stdout, cli.Show.Bucket, cli.Show.Path, cli.Show.HeaderJson, cli.Show.Metadata, cli.Show.Tilejson, cli.Show.PublicURL, false, 0, 0, 0)
 		if err != nil {
 			logger.Fatalf("Failed to show archive, %v", err)
 		}
 	case "tile <path> <z> <x> <y>":
-		err := pmtiles.Show(logger, cli.Tile.Bucket, cli.Tile.Path, false, false, "", true, cli.Tile.Z, cli.Tile.X, cli.Tile.Y)
+		err := pmtiles.Show(logger, os.Stdout, cli.Tile.Bucket, cli.Tile.Path, false, false, false, "", true, cli.Tile.Z, cli.Tile.X, cli.Tile.Y)
 		if err != nil {
 			logger.Fatalf("Failed to show tile, %v", err)
 		}

pmtiles/directory.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"encoding/binary"
+	"encoding/json"
 	"fmt"
 )
 
@@ -63,6 +64,23 @@ type HeaderV3 struct {
 	CenterLatE7         int32
 }
 
+// HeaderJson is a human-readable representation of parts of the binary header
+// that may need to be manually edited.
+// Omitted parts are the responsibility of the generator program and not editable.
+type HeaderJson struct {
+	TileCompression string
+	TileType        string
+	MinZoom         int
+	MaxZoom         int
+	MinLon          float64
+	MinLat          float64
+	MaxLon          float64
+	MaxLat          float64
+	CenterZoom      int
+	CenterLon       float64
+	CenterLat       float64
+}
+
 func headerContentType(header HeaderV3) (string, bool) {
 	switch header.TileType {
 	case Mvt:
@@ -80,23 +98,31 @@ func headerContentType(header HeaderV3) (string, bool) {
 	}
 }
 
-func headerExt(header HeaderV3) string {
-	switch header.TileType {
+func stringifiedTileType(t TileType) string {
+	switch t {
 	case Mvt:
-		return ".mvt"
+		return "mvt"
 	case Png:
-		return ".png"
+		return "png"
 	case Jpeg:
-		return ".jpg"
+		return "jpg"
 	case Webp:
-		return ".webp"
+		return "webp"
 	case Avif:
-		return ".avif"
+		return "avif"
 	default:
 		return ""
 	}
 }
 
+func headerExt(header HeaderV3) string {
+	base := stringifiedTileType(header.TileType)
+	if base == "" {
+		return ""
+	}
+	return "." + base
+}
+
 func headerContentEncoding(compression Compression) (string, bool) {
 	switch compression {
 	case Gzip:
@@ -108,6 +134,28 @@ func headerContentEncoding(compression Compression) (string, bool) {
 	}
 }
 
+func headerToJson(header HeaderV3) HeaderJson {
+	compressionString, _ := headerContentEncoding(header.TileCompression)
+	return HeaderJson{
+		TileCompression: compressionString,
+		TileType:        stringifiedTileType(header.TileType),
+		MinZoom:         int(header.MinZoom),
+		MaxZoom:         int(header.MaxZoom),
+		MinLon:          float64(header.MinLonE7) / 10000000,
+		MinLat:          float64(header.MinLatE7) / 10000000,
+		MaxLon:          float64(header.MaxLonE7) / 10000000,
+		MaxLat:          float64(header.MaxLatE7) / 10000000,
+		CenterZoom:      int(header.CenterZoom),
+		CenterLon:       float64(header.CenterLonE7) / 10000000,
+		CenterLat:       float64(header.CenterLatE7) / 10000000,
+	}
+}
+
+func headerToStringifiedJson(header HeaderV3) string {
+	s, _ := json.MarshalIndent(headerToJson(header), "", "    ")
+	return string(s)
+}
+
 // EntryV3 is an entry in a PMTiles spec version 3 directory.
 type EntryV3 struct {
 	TileID    uint64

pmtiles/directory_test.go

@@ -84,6 +84,33 @@ func TestHeaderRoundtrip(t *testing.T) {
 	assert.Equal(t, int32(32000000), result.CenterLatE7)
 }
 
+func TestHeaderJsonRoundtrip(t *testing.T) {
+	header := HeaderV3{}
+	header.TileCompression = Brotli
+	header.TileType = Mvt
+	header.MinZoom = 1
+	header.MaxZoom = 3
+	header.MinLonE7 = 1.1 * 10000000
+	header.MinLatE7 = 2.1 * 10000000
+	header.MaxLonE7 = 1.2 * 10000000
+	header.MaxLatE7 = 2.2 * 10000000
+	header.CenterZoom = 2
+	header.CenterLonE7 = 3.1 * 10000000
+	header.CenterLatE7 = 3.2 * 10000000
+	j := headerToJson(header)
+	assert.Equal(t, "br", j.TileCompression)
+	assert.Equal(t, "mvt", j.TileType)
+	assert.Equal(t, 1, j.MinZoom)
+	assert.Equal(t, 3, j.MaxZoom)
+	assert.Equal(t, 2, j.CenterZoom)
+	assert.Equal(t, 1.1, j.MinLon)
+	assert.Equal(t, 2.1, j.MinLat)
+	assert.Equal(t, 1.2, j.MaxLon)
+	assert.Equal(t, 2.2, j.MaxLat)
+	assert.Equal(t, 3.1, j.CenterLon)
+	assert.Equal(t, 3.2, j.CenterLat)
+}
+
 func TestOptimizeDirectories(t *testing.T) {
 	rand.Seed(3857)
 	entries := make([]EntryV3, 0)
@@ -171,3 +198,12 @@ func TestBuildRootsLeaves(t *testing.T) {
 	_, _, numLeaves := buildRootsLeaves(entries, 1)
 	assert.Equal(t, 1, numLeaves)
 }
+
+func TestStringifiedExtension(t *testing.T) {
+	assert.Equal(t, "", headerExt(HeaderV3{}))
+	assert.Equal(t, ".mvt", headerExt(HeaderV3{TileType: Mvt}))
+	assert.Equal(t, ".png", headerExt(HeaderV3{TileType: Png}))
+	assert.Equal(t, ".jpg", headerExt(HeaderV3{TileType: Jpeg}))
+	assert.Equal(t, ".webp", headerExt(HeaderV3{TileType: Webp}))
+	assert.Equal(t, ".avif", headerExt(HeaderV3{TileType: Avif}))
+}

pmtiles/extract_test.go

@@ -1,7 +1,6 @@
 package pmtiles
 
 import (
-	"fmt"
 	"github.com/RoaringBitmap/roaring/roaring64"
 	"github.com/stretchr/testify/assert"
 	"testing"
@@ -164,5 +163,4 @@ func TestMergeRangesMultiple(t *testing.T) {
 	assert.Equal(t, 1, result.Len())
 	assert.Equal(t, srcDstRange{0, 0, 90}, front.Rng)
 	assert.Equal(t, 3, len(front.CopyDiscards))
-	fmt.Println(result)
 }

pmtiles/show.go

@@ -14,7 +14,7 @@ import (
 )
 
 // Show prints detailed information about an archive.
-func Show(_ *log.Logger, bucketURL string, key string, showMetadataOnly bool, showTilejson bool, publicURL string, showTile bool, z int, x int, y int) error {
+func Show(_ *log.Logger, output io.Writer, bucketURL string, key string, showHeaderJsonOnly bool, showMetadataOnly bool, showTilejson bool, publicURL string, showTile bool, z int, x int, y int) error {
 	ctx := context.Background()
 
 	bucketURL, key, err := NormalizeBucketKey(bucketURL, "", key)
@@ -90,11 +90,13 @@ func Show(_ *log.Logger, bucketURL string, key string, showMetadataOnly bool, sh
 		metadataReader.Close()
 
 		if showMetadataOnly && showTilejson {
-			return fmt.Errorf("cannot use --metadata and --tilejson together")
+			return fmt.Errorf("cannot use more than one of --header-json, --metadata, and --tilejson together")
 		}
 
-		if showMetadataOnly {
-			fmt.Print(string(metadataBytes))
+		if showHeaderJsonOnly {
+			fmt.Fprintln(output, headerToStringifiedJson(header))
+		} else if showMetadataOnly {
+			fmt.Fprintln(output, string(metadataBytes))
 		} else if showTilejson {
 			if publicURL == "" {
 				// Using Fprintf instead of logger here, as this message should be written to Stderr in case
@@ -105,7 +107,7 @@ func Show(_ *log.Logger, bucketURL string, key string, showMetadataOnly bool, sh
 			if err != nil {
 				return fmt.Errorf("Failed to create tilejson for %s, %w", key, err)
 			}
-			fmt.Print(string(tilejsonBytes))
+			fmt.Fprintln(output, string(tilejsonBytes))
 		} else {
 			fmt.Printf("pmtiles spec version: %d\n", header.SpecVersion)
 			// fmt.Printf("total size: %s\n", humanize.Bytes(uint64(r.Size())))
@@ -164,7 +166,7 @@ func Show(_ *log.Logger, bucketURL string, key string, showMetadataOnly bool, sh
 					if err != nil {
 						return fmt.Errorf("I/O Error")
 					}
-					os.Stdout.Write(tileBytes)
+					output.Write(tileBytes)
 					break
 				}
 				dirOffset = header.LeafDirectoryOffset + entry.Offset

pmtiles/show_test.go

@@ -0,0 +1,33 @@
+package pmtiles
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/stretchr/testify/assert"
+	"log"
+	"os"
+	"testing"
+)
+
+func TestShowHeader(t *testing.T) {
+	var b bytes.Buffer
+	logger := log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)
+	err := Show(logger, &b, "", "fixtures/test_fixture_1.pmtiles", true, false, false, "", false, 0, 0, 0)
+	assert.Nil(t, err)
+
+	var input map[string]interface{}
+	json.Unmarshal(b.Bytes(), &input)
+	assert.Equal(t, "mvt", input["TileType"])
+	assert.Equal(t, "gzip", input["TileCompression"])
+}
+
+func TestShowMetadata(t *testing.T) {
+	var b bytes.Buffer
+	logger := log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)
+	err := Show(logger, &b, "", "fixtures/test_fixture_1.pmtiles", false, true, false, "", false, 0, 0, 0)
+	assert.Nil(t, err)
+
+	var input map[string]interface{}
+	json.Unmarshal(b.Bytes(), &input)
+	assert.Equal(t, "tippecanoe v2.5.0", input["generator"])
+}

pmtiles/write.go

@@ -0,0 +1,42 @@
+package pmtiles
+
+import (
+	"fmt"
+	"log"
+	"os"
+)
+
+func Write(logger *log.Logger, inputArchive string, newHeaderJsonFile string, newMetadataFile string) error {
+	if newMetadataFile == "" {
+		if newHeaderJsonFile == "" {
+			return fmt.Errorf("No data to write.")
+		}
+		// we can write the header in-place without writing the whole file.
+		return nil
+	}
+
+	// write metadata:
+	// always writes in this order:
+	// copy the header
+	// copy the root directory
+	// write the new the metadata
+	// copy the leaf directories
+	// copy the tile data
+	file, err := os.OpenFile(inputArchive, os.O_RDWR, 0666)
+
+	buf := make([]byte, 127)
+	_, err = file.Read(buf)
+	if err != nil {
+		return err
+	}
+	originalHeader, _ := deserializeHeader(buf)
+
+	// modify the header
+
+	buf = serializeHeader(originalHeader)
+	_, err = file.WriteAt(buf, 0)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pmtiles/write_test.go

@@ -0,0 +1,19 @@
+package pmtiles
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestWriteHeader(t *testing.T) {
+	tempDir, _ := ioutil.TempDir("", "testing")
+	defer os.RemoveAll(tempDir)
+	src, _ := os.Open("fixtures/test_fixture_1.pmtiles")
+	defer src.Close()
+	dest, _ := os.Create(filepath.Join(tempDir, "test.pmtiles"))
+	defer dest.Close()
+	_, _ = io.Copy(dest, src)
+}