librclone: exports, errors, docs and examples #4891

- rename C exports to be namespaced with Rclone prefix
- fix error handling in RcloneRPC
- add more examples
- add more docs
- add README
- simplify ctest Makefile

librclone/README.md

@@ -0,0 +1,31 @@
+# librclone
+
+This directory contains code to build rclone as a C library and the
+shims for accessing rclone from C.
+
+The shims are a thin wrapper over the rclone RPC.
+
+Build a shared library like this:
+
+    go build --buildmode=c-shared -o librclone.so github.com/rclone/rclone/librclone
+
+Build a static library like this:
+
+    go build --buildmode=c-archive -o librclone.a github.com/rclone/rclone/librclone
+
+Both the above commands will also generate `librclone.h` which should
+be `#include`d in `C` programs wishing to use the library.
+
+The library will depend on `libdl` and `libpthread`.
+
+## Documentation
+
+For documentation see the Go documentation for:
+
+- [RcloneInitialize](https://pkg.go.dev/github.com/rclone/rclone/librclone#RcloneInitialize)
+- [RcloneFinalize](https://pkg.go.dev/github.com/rclone/rclone/librclone#RcloneFinalize)
+- [RcloneRPC](https://pkg.go.dev/github.com/rclone/rclone/librclone#RcloneRPC)
+
+## C Example
+
+There is an example program `ctest.c` with Makefile in the `ctest` subdirectory

librclone/ctest/Makefile

@@ -1,21 +1,13 @@
 CFLAGS = -g -Wall
 LDFLAGS = -L. -lrclone -lpthread -ldl
 
-static: ctest
+ctest: ctest.o librclone.a
-
-shared:
-	go build --buildmode=c-shared -o librclone.so github.com/rclone/rclone/librclone
-
-ctest: ctest.o librclone.h
 	$(CC) $(CFLAGS) -o $@ $< $(LDFLAGS)
 
 ctest.o: ctest.c librclone.h
 	$(CC) $(CFLAGS) -c $^ $(LDFLAGS)
 
-build:
+librclone.a librclone.h:
-	go build
-
-librclone.h:
 	go build --buildmode=c-archive -o librclone.a github.com/rclone/rclone/librclone
 
 clean:

librclone/ctest/ctest.c

@@ -4,29 +4,60 @@
 #include <dlfcn.h>
 #include "librclone.h"
 
-// copy file using "operations/copyfile" command
+void testRPC(char *method, char *in) {
-void testCopyFile() {
+    struct RcloneRPC_return out = RcloneRPC(method, in);
-    struct CRPC_return res = CRPC("operations/copyfile", "{ \"srcFs\": \"/tmp\", \"srcRemote\": \"tmpfile\", \"dstFs\": \"/tmp\", \"dstRemote\": \"tmpfile2\" }");
+    printf("status: %d\n", out.r1);
-    printf("%d\n", res.r1); // status
+    printf("output: %s\n", out.r0);
-    printf("%s\n", res.r0); // output
+    free(out.r0);
-    free(res.r0);
 }
 
 // noop command
 void testNoOp() {
-    struct CRPC_return res = CRPC("rc/noop", "{ \"p1\": [1,\"2\",null,4], \"p2\": { \"a\":1, \"b\":2 } }");
+    printf("test rc/noop\n");
-    printf("%d\n", res.r1); // status
+    testRPC("rc/noop",
-    printf("%s\n", res.r0); // output
+            "{"
-    free(res.r0);
+            " \"p1\": [1,\"2\",null,4],"
+            " \"p2\": { \"a\":1, \"b\":2 } "
+            "}");
+}
+
+// error command
+void testError() {
+    printf("test rc/error\n");
+    testRPC("rc/error",
+            "{"
+            " \"p1\": [1,\"2\",null,4],"
+            " \"p2\": { \"a\":1, \"b\":2 } "
+            "}");
+}
+
+// copy file using "operations/copyfile" command
+void testCopyFile() {
+    printf("test operations/copyfile\n");
+    testRPC("operations/copyfile",
+            "{"
+                "\"srcFs\": \"/tmp\","
+                "\"srcRemote\": \"tmpfile\","
+                "\"dstFs\": \"/tmp\","
+                "\"dstRemote\": \"tmpfile2\""
+            "}");
+}
+
+// list the remotes
+void testListRemotes() {
+    printf("test operations/listremotes\n");
+    testRPC("config/listremotes", "{}");
 }
 
 int main(int argc, char** argv) {
     printf("c main begin\n");
-    Cinit();
+    RcloneInitialize();
 
-    //testNoOp();
+    testNoOp();
+    testError();
     testCopyFile();
+    testListRemotes();
 
-    Cdestroy();
+    RcloneFinalize();
     return EXIT_SUCCESS;
 }

librclone/librclone.go

@@ -1,13 +1,30 @@
-// Package exports exports function for c library
+// Package librclone exports shims for C library use
-
+//
+// This directory contains code to build rclone as a C library and the
+// shims for accessing rclone from C.
+//
+// The shims are a thin wrapper over the rclone RPC.
+//
+// Build a shared library like this:
+//
+//     go build --buildmode=c-shared -o librclone.so github.com/rclone/rclone/librclone
+//
+// Build a static library like this:
+//
+//     go build --buildmode=c-archive -o librclone.a github.com/rclone/rclone/librclone
+//
+// Both the above commands will also generate `librclone.h` which should
+// be `#include`d in `C` programs wishing to use the library.
+//
+// The library will depend on `libdl` and `libpthread`.
 package main
 
 import (
 	"C"
-	"bytes"
+
 	"context"
 	"encoding/json"
-	"io"
+	"fmt"
 	"net/http"
 	"runtime"
 	"strings"
@@ -18,40 +35,51 @@ import (
 	"github.com/rclone/rclone/fs/rc/jobs"
 
 	_ "github.com/rclone/rclone/backend/all" // import all backends
-	_ "github.com/rclone/rclone/cmd/all"     // import all commands
 	_ "github.com/rclone/rclone/lib/plugin"  // import plugins
 )
 
-func init() {
+// RcloneInitialize initializes rclone as a library
-	// do nothing
+//
-}
+//export RcloneInitialize
-
+func RcloneInitialize() {
-// call to init the library
-//export Cinit
-func Cinit() {
 	// TODO: what need to be initialized manually?
 }
 
-// call to destroy the whole thing
+// RcloneFinalize finalizes the library
-//export Cdestroy
+//
-func Cdestroy() {
+//export RcloneFinalize
+func RcloneFinalize() {
 	// TODO: how to clean up? what happens when rcserver terminates?
 	// what about unfinished async jobs?
 	runtime.GC()
 }
 
+// RcloneRPC does a single RPC call. The inputs are (method, input)
+// and the output is (output, status). This is an exported interface
+// to the rclone API as described in https://rclone.org/rc/
+//
+//   method is a string, eg "operations/list"
+//   input should be a serialized JSON object
+//   output will be returned as a serialized JSON object
+//   status is a HTTP status return (200=OK anything else fail)
+//
 // Caller is responsible for freeing the memory for output
-// TODO: how to specify config file?
+//
-//export CRPC
+// Note that when calling from C output and status are returned in an
-func CRPC(method *C.char, input *C.char) (output *C.char, status C.int) {
+// RcloneRPC_return which has two members r0 which is output and r1
+// which is status.
+//
+//export RcloneRPC
+func RcloneRPC(method *C.char, input *C.char) (output *C.char, status C.int) { //nolint:golint
 	res, s := callFunctionJSON(C.GoString(method), C.GoString(input))
 	return C.CString(res), C.int(s)
 }
 
-// copied from rcserver.go
+// writeError returns a formatted error string and the status passed in
-// writeError writes a formatted error to the output
+func writeError(path string, in rc.Params, err error, status int) (string, int) {
-func writeError(path string, in rc.Params, w io.Writer, err error, status int) {
 	fs.Errorf(nil, "rc: %q: error: %v", path, err)
+	var w strings.Builder
+	// FIXME should factor this
 	// Adjust the error return for some well known errors
 	errOrig := errors.Cause(err)
 	switch {
@@ -61,7 +89,7 @@ func writeError(path string, in rc.Params, w io.Writer, err error, status int) {
 		status = http.StatusBadRequest
 	}
 	// w.WriteHeader(status)
-	err = rc.WriteJSON(w, rc.Params{
+	err = rc.WriteJSON(&w, rc.Params{
 		"status": status,
 		"error":  err.Error(),
 		"input":  in,
@@ -69,8 +97,9 @@ func writeError(path string, in rc.Params, w io.Writer, err error, status int) {
 	})
 	if err != nil {
 		// can't return the error at this point
-		fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
+		return fmt.Sprintf(`{"error": "rc: failed to write JSON output: %v"}`, err), status
 	}
+	return w.String(), status
 }
 
 // operations/uploadfile and core/command are not supported as they need request or response object
@@ -78,32 +107,27 @@ func writeError(path string, in rc.Params, w io.Writer, err error, status int) {
 // call a rc function using JSON to input parameters and output the resulted JSON
 func callFunctionJSON(method string, input string) (output string, status int) {
 	// create a buffer to capture the output
-	buf := new(bytes.Buffer)
 	in := make(rc.Params)
 	err := json.NewDecoder(strings.NewReader(input)).Decode(&in)
 	if err != nil {
 		// TODO: handle error
-		writeError(method, in, buf, errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest)
+		return writeError(method, in, errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest)
-		return buf.String(), http.StatusBadRequest
 	}
 
 	// Find the call
 	call := rc.Calls.Get(method)
 	if call == nil {
-		writeError(method, in, buf, errors.Errorf("couldn't find method %q", method), http.StatusNotFound)
+		return writeError(method, in, errors.Errorf("couldn't find method %q", method), http.StatusNotFound)
-		return buf.String(), http.StatusNotFound
 	}
 
 	// TODO: handle these cases
 	if call.NeedsRequest {
-		writeError(method, in, buf, errors.Errorf("method %q needs request, not supported", method), http.StatusNotFound)
+		return writeError(method, in, errors.Errorf("method %q needs request, not supported", method), http.StatusNotFound)
-		return buf.String(), http.StatusNotFound
 		// Add the request to RC
 		//in["_request"] = r
 	}
 	if call.NeedsResponse {
-		writeError(method, in, buf, errors.Errorf("method %q need response, not supported", method), http.StatusNotFound)
+		return writeError(method, in, errors.Errorf("method %q need response, not supported", method), http.StatusNotFound)
-		return buf.String(), http.StatusNotFound
 		//in["_response"] = w
 	}
 
@@ -112,22 +136,22 @@ func callFunctionJSON(method string, input string) (output string, status int) {
 	_, out, err := jobs.NewJob(context.Background(), call.Fn, in)
 	if err != nil {
 		// handle error
-		writeError(method, in, buf, err, http.StatusInternalServerError)
+		return writeError(method, in, err, http.StatusInternalServerError)
-		return buf.String(), http.StatusInternalServerError
 	}
 	if out == nil {
 		out = make(rc.Params)
 	}
 
 	fs.Debugf(nil, "rc: %q: reply %+v: %v", method, out, err)
-	err = rc.WriteJSON(buf, out)
+
+	var w strings.Builder
+	err = rc.WriteJSON(&w, out)
 	if err != nil {
-		writeError(method, in, buf, err, http.StatusInternalServerError)
-		return buf.String(), http.StatusInternalServerError
 		fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
+		return writeError(method, in, err, http.StatusInternalServerError)
 	}
-	return buf.String(), http.StatusOK
+	return w.String(), http.StatusOK
 }
 
-// do nothing here
+// do nothing here - necessary for building into a C library
 func main() {}