update 1.5 sync urls #259

api/api.go

@@ -1,6 +1,8 @@
 package api
 
 import (
+	"fmt"
+	"os"
 	"strings"
 
 	"github.com/golang-jwt/jwt"
@@ -28,7 +30,7 @@ type UserToken struct {
 }
 
 // CreateApiCtx initializes an instance of ApiCtx
-func CreateApiCtx(http *transport.HttpClientCtx) (ctx ApiCtx, isSync15 bool, err error) {
+func CreateApiCtx(http *transport.HttpClientCtx) (ctx ApiCtx, err error) {
 	userToken := http.Tokens.UserToken
 	claims := UserToken{}
 	jwt.ParseWithClaims(userToken, &claims, func(token *jwt.Token) (interface{}, error) {
@@ -38,6 +40,8 @@ func CreateApiCtx(http *transport.HttpClientCtx) (ctx ApiCtx, isSync15 bool, err
 		return
 	}
 	fld := strings.Fields(claims.Scopes)
+	isSync15 := false
+
 forloop:
 	for _, f := range fld {
 		switch f {
@@ -51,6 +55,7 @@ forloop:
 		}
 	}
 	if isSync15 {
+		fmt.Fprintln(os.Stderr, `WARNING!!! Using the new 1.5 sync, this has not been fully tested yet!!! first sync can be slow`)
 		ctx, err = sync15.CreateCtx(http)
 	} else {
 		ctx, err = sync10.CreateCtx(http)

api/sync10/api.go

@@ -37,7 +37,7 @@ func (ctx *ApiCtx) Nuke() error {
 	for _, d := range documents {
 		log.Info.Println("Deleting: ", d.VissibleName)
 
-		err := ctx.Http.Put(transport.UserBearer, config.DeleteEntry, d, nil)
+		err := ctx.Http.Put(transport.UserBearer, config.DeleteEntry, util.InSlice(d), nil)
 		if err != nil {
 			log.Error.Println("failed to remove entry", err)
 			return err
@@ -141,7 +141,7 @@ func (ctx *ApiCtx) CreateDir(parentId, name string) (*model.Document, error) {
 
 	metaDoc := model.CreateUploadDocumentMeta(uploadRsp.ID, model.DirectoryType, parentId, name)
 
-	err = ctx.Http.Put(transport.UserBearer, config.UpdateStatus, metaDoc, nil)
+	err = ctx.Http.Put(transport.UserBearer, config.UpdateStatus, util.InSlice(metaDoc), nil)
 
 	if err != nil {
 		log.Error.Println("failed to move entry", err)
@@ -162,7 +162,7 @@ func (ctx *ApiCtx) DeleteEntry(node *model.Node) error {
 
 	deleteDoc := node.Document.ToDeleteDocument()
 
-	err := ctx.Http.Put(transport.UserBearer, config.DeleteEntry, deleteDoc, nil)
+	err := ctx.Http.Put(transport.UserBearer, config.DeleteEntry, util.InSlice(deleteDoc), nil)
 
 	if err != nil {
 		log.Error.Println("failed to remove entry", err)
@@ -186,7 +186,7 @@ func (ctx *ApiCtx) MoveEntry(src, dstDir *model.Node, name string) (*model.Node,
 	metaDoc.VissibleName = name
 	metaDoc.Parent = dstDir.Id()
 
-	err := ctx.Http.Put(transport.UserBearer, config.UpdateStatus, metaDoc, nil)
+	err := ctx.Http.Put(transport.UserBearer, config.UpdateStatus, util.InSlice(metaDoc), nil)
 
 	if err != nil {
 		log.Error.Println("failed to move entry", err)
@@ -258,7 +258,7 @@ func (ctx *ApiCtx) UploadDocument(parentId string, sourceDocPath string, notify
 
 	metaDoc := model.CreateUploadDocumentMeta(uploadRsp.ID, model.DocumentType, parentId, name)
 
-	err = ctx.Http.Put(transport.UserBearer, config.UpdateStatus, metaDoc, nil)
+	err = ctx.Http.Put(transport.UserBearer, config.UpdateStatus, util.InSlice(metaDoc), nil)
 
 	if err != nil {
 		log.Error.Println("failed to move entry", err)
@@ -274,7 +274,7 @@ func (ctx *ApiCtx) uploadRequest(id string, entryType string) (model.UploadDocum
 	uploadReq := model.CreateUploadDocumentRequest(id, entryType)
 	uploadRsp := make([]model.UploadDocumentResponse, 0)
 
-	err := ctx.Http.Put(transport.UserBearer, config.UploadRequest, uploadReq, &uploadRsp)
+	err := ctx.Http.Put(transport.UserBearer, config.UploadRequest, util.InSlice(uploadReq), &uploadRsp)
 
 	if err != nil {
 		log.Error.Println("failed to to send upload request", err)

api/sync15/apictx.go

@@ -196,9 +196,8 @@ func Sync(b *BlobStorage, tree *HashTree, operation func(t *HashTree) error) err
 
 		newGeneration, err := b.WriteRootIndex(tree.Hash, tree.Generation)
 
-		log.Info.Println("wrote root, new gen: ", newGeneration)
-
 		if err == nil {
+			log.Info.Println("wrote root, new gen: ", newGeneration)
 			tree.Generation = newGeneration
 			break
 		}
@@ -388,7 +387,6 @@ func CreateCtx(http *transport.HttpClientCtx) (*ApiCtx, error) {
 		fmt.Print(err)
 		return nil, err
 	}
-	fmt.Fprintln(os.Stderr, "Refreshing tree...")
 	err = cacheTree.Mirror(apiStorage)
 	if err != nil {
 		return nil, err
@@ -426,7 +424,7 @@ func DocumentsFileTree(tree *HashTree) (*filetree.FileTreeCtx, error) {
 
 // SyncComplete notfies that somethings has changed (triggers tablet sync)
 func (ctx *ApiCtx) SyncComplete() error {
-	err := ctx.blobStorage.SyncComplete()
+	err := ctx.blobStorage.SyncComplete(ctx.hashTree.Generation)
 	if err != nil {
 		log.Error.Printf("cannot send sync %v", err)
 	}

api/sync15/blobdoc.go

@@ -7,7 +7,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"sort"
@@ -218,7 +217,6 @@ func (d *BlobDoc) ToDocument() *model.Document {
 
 		t := time.Unix(unixTime/1000, 0)
 		lastModified = t.UTC().Format(time.RFC3339Nano)
-		fmt.Println(" Time " + lastModified + " " + d.MetadataFile.DocName)
 	}
 	return &model.Document{
 		ID:             d.DocumentID,

api/sync15/blobstorage.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"io"
 	"io/ioutil"
-	"strconv"
+	"net/http"
 
 	"github.com/juruen/rmapi/config"
 	"github.com/juruen/rmapi/log"
@@ -16,26 +16,40 @@ type BlobStorage struct {
 	http *transport.HttpClientCtx
 }
 
-func (b *BlobStorage) PutUrl(hash string, gen int64) (string, error) {
+const ROOT_NAME = "root"
+
+func (b *BlobStorage) PutRootUrl(hash string, gen int64) (string, int64, error) {
+	log.Trace.Println("fetching  ROOT url for: " + hash)
+	req := model.BlobRootStorageRequest{
+		Method:       http.MethodPut,
+		RelativePath: ROOT_NAME,
+		RootSchema:   hash,
+		Generation:   gen,
+	}
+	var res model.BlobStorageResponse
+
+	if err := b.http.Post(transport.UserBearer, config.UploadBlob, req, &res); err != nil {
+		return "", 0, err
+	}
+	return res.Url, res.MaxUploadSizeBytes, nil
+}
+func (b *BlobStorage) PutUrl(hash string) (string, int64, error) {
 	log.Trace.Println("fetching PUT blob url for: " + hash)
 	var req model.BlobStorageRequest
 	var res model.BlobStorageResponse
-	req.Method = "PUT"
+	req.Method = http.MethodPut
 	req.RelativePath = hash
-	if gen > 0 {
-		req.Generation = strconv.FormatInt(gen, 10)
-	}
 	if err := b.http.Post(transport.UserBearer, config.UploadBlob, req, &res); err != nil {
-		return "", err
+		return "", 0, err
 	}
-	return res.Url, nil
+	return res.Url, res.MaxUploadSizeBytes, nil
 }
 
 func (b *BlobStorage) GetUrl(hash string) (string, error) {
 	log.Trace.Println("fetching GET blob url for: " + hash)
 	var req model.BlobStorageRequest
 	var res model.BlobStorageResponse
-	req.Method = "GET"
+	req.Method = http.MethodGet
 	req.RelativePath = hash
 	if err := b.http.Post(transport.UserBearer, config.DownloadBlob, req, &res); err != nil {
 		return "", err
@@ -55,37 +69,36 @@ func (b *BlobStorage) GetReader(hash string) (io.ReadCloser, error) {
 }
 
 func (b *BlobStorage) UploadBlob(hash string, reader io.Reader) error {
-	url, err := b.PutUrl(hash, -1)
+	url, size, err := b.PutUrl(hash)
 	if err != nil {
 		return err
 	}
 	log.Trace.Println("put url: " + url)
 
-	_, err = b.http.PutBlobStream(url, -1, reader)
-	return err
+	return b.http.PutBlobStream(url, reader, size)
 }
 
 // SyncComplete notifies that the sync is done
-func (b *BlobStorage) SyncComplete() error {
-	return b.http.Post(transport.UserBearer, config.SyncComplete, nil, nil)
+func (b *BlobStorage) SyncComplete(gen int64) error {
+	req := model.SyncCompletedRequest{
+		Generation: gen,
+	}
+	return b.http.Post(transport.UserBearer, config.SyncComplete, req, nil)
 }
 
 func (b *BlobStorage) WriteRootIndex(roothash string, gen int64) (int64, error) {
 	log.Info.Println("writing root with gen: ", gen)
-	url, err := b.PutUrl("root", gen)
+	url, maxRequestSize, err := b.PutRootUrl(roothash, gen)
 	if err != nil {
 		return 0, err
 	}
 	log.Trace.Println("got root url:", url)
 	reader := bytes.NewBufferString(roothash)
 
-	gen, err = b.http.PutBlobStream(url, gen, reader)
-	return gen, err
-
+	return b.http.PutRootBlobStream(url, gen, maxRequestSize, reader)
 }
 func (b *BlobStorage) GetRootIndex() (string, int64, error) {
-
-	url, err := b.GetUrl("root")
+	url, err := b.GetUrl(ROOT_NAME)
 	if err != nil {
 		return "", 0, err
 	}

api/sync15/tree.go

@@ -161,7 +161,7 @@ func (t *HashTree) Rehash() error {
 	return nil
 }
 
-/// Mirror makes the tree look like the storage
+// / Mirror makes the tree look like the storage
 func (t *HashTree) Mirror(r RemoteStorage) error {
 	rootHash, gen, err := r.GetRootIndex()
 	if err != nil {

cloud/client_test.go

@@ -1,3 +1,4 @@
+//go:build withauth
 // +build withauth
 
 package cloud

config/url.go

@@ -16,7 +16,7 @@ var SyncComplete string
 func init() {
 	docHost := "https://document-storage-production-dot-remarkable-production.appspot.com"
 	authHost := "https://webapp-production-dot-remarkable-production.appspot.com"
-	blobHost := "https://rm-blob-storage-prod.appspot.com"
+	syncHost := "https://internal.cloud.remarkable.com"
 
 	host := os.Getenv("RMAPI_DOC")
 	if host != "" {
@@ -33,7 +33,7 @@ func init() {
 	if host != "" {
 		authHost = host
 		docHost = host
-		blobHost = host
+		syncHost = host
 	}
 
 	NewTokenDevice = authHost + "/token/json/2/device/new"
@@ -43,7 +43,7 @@ func init() {
 	UploadRequest = docHost + "/document-storage/json/2/upload/request"
 	DeleteEntry = docHost + "/document-storage/json/2/delete"
 
-	UploadBlob = blobHost + "/api/v1/signed-urls/uploads"
-	DownloadBlob = blobHost + "/api/v1/signed-urls/downloads"
-	SyncComplete = blobHost + "/api/v1/sync-complete"
+	UploadBlob = syncHost + "/sync/v2/signed-urls/uploads"
+	DownloadBlob = syncHost + "/sync/v2/signed-urls/downloads"
+	SyncComplete = syncHost + "/sync/v2/sync-complete"
 }

encoding/rm/rm.go

@@ -23,8 +23,8 @@
 //
 // To try to be as idiomatic as possible, this package implements the two following interfaces
 // of the default encoding package (https://golang.org/pkg/encoding/).
-//  - BinaryMarshaler
-//  - BinaryUnmarshaler
+//   - BinaryMarshaler
+//   - BinaryUnmarshaler
 //
 // The scope of this package is defined as just the encoding/decoding of the .rm format.
 // It will only deal with bytes and not files (one must take care of unzipping the archive

main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"os"
 
 	"github.com/juruen/rmapi/api"
@@ -31,17 +30,11 @@ func main() {
 	var ctx api.ApiCtx
 	var err error
 	for i := 0; i < AUTH_RETRIES; i++ {
-		isSync15 := false
-		ctx, isSync15, err = api.CreateApiCtx(api.AuthHttpCtx(i > 0, *ni))
+		ctx, err = api.CreateApiCtx(api.AuthHttpCtx(i > 0, *ni))
 
 		if err != nil {
 			log.Trace.Println(err)
 		} else {
-			if isSync15 {
-				fmt.Fprintln(os.Stderr, `WARNING!!!
-  Using the new 1.5 sync, this has not been fully tested yet!!!
-  Make sure you have a backup, in case there is a bug that could cause data loss!`)
-			}
 			break
 		}
 	}

model/document.go

@@ -56,20 +56,34 @@ type UploadDocumentResponse struct {
 	BlobURLPutExpires string
 }
 
-// BlobStorageRequest else
+type BlobRootStorageRequest struct {
+	Method       string `json:"http_method"`
+	Initial      bool   `json:"initial_sync,omitempty"`
+	RelativePath string `json:"relative_path"`
+	RootSchema   string `json:"root_schema,omitempty"`
+	Generation   int64  `json:"generation"`
+}
+
+// BlobStorageRequest request
 type BlobStorageRequest struct {
 	Method       string `json:"http_method"`
 	Initial      bool   `json:"initial_sync,omitempty"`
 	RelativePath string `json:"relative_path"`
-	Generation   string `json:"generation,omitempty"`
+	ParentPath   string `json:"parent_path,omitempty"`
 }
 
-// BlobStorageResponse  what else
+// BlobStorageResponse response
 type BlobStorageResponse struct {
-	Expires      string `json:"expires"`
-	Method       string `json:"method"`
-	RelativePath string `json:"relative_path"`
-	Url          string `json:"url"`
+	Expires            string `json:"expires"`
+	Method             string `json:"method"`
+	RelativePath       string `json:"relative_path"`
+	Url                string `json:"url"`
+	MaxUploadSizeBytes int64  `json:"maxuploadsize_bytes,omitifempty"`
+}
+
+// SyncCompleteRequest payload of the sync completion
+type SyncCompletedRequest struct {
+	Generation int64 `json:"generation"`
 }
 
 func CreateDirDocument(parent, name string) MetadataDocument {

shell/mput.go

@@ -69,10 +69,11 @@ func mputCmd(ctx *ShellCtxt) *ishell.Cmd {
 // Print the required spaces and characters for tree formatting.
 //
 // Input -> [*ishell.Context]
-//			[int]				tree depth (0 ... N-1)
-//			[int]				Current item index in directory
-//			[int]				Current directory list length
-//			[*string]			Book keeping for tree formatting
+//
+//	[int]				tree depth (0 ... N-1)
+//	[int]				Current item index in directory
+//	[int]				Current directory list length
+//	[*string]			Book keeping for tree formatting
 func treeFormat(pC *ishell.Context, num int, lIndex int, lSize int, tFS *string) {
 
 	tFStr := ""