commit 1190775e94152ca0093b44d35dcb927b714112b5 Author: cellarspoon Date: Tue Feb 1 00:08:55 2022 +0100 init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9daeafb --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +test diff --git a/README.md b/README.md new file mode 100644 index 0000000..dc0080c --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# distribusi-go + +> This is still very experimental, please take a backup of your archives if you +> are running it on files you care about. It hasn't been tested on large +> archives. It may still thrash files. Please [report issues] as you find them +> :100: + +A [Go] implementation of [distribusi]. + +This is a compiled `distribusi` which is simpler to install on your computer, +just download the binary, `chmod +x` and run it. + +The command-line interface is quite different from the Python version. There +are less optional flags and more defaults. I shuffled a number of things around +according to my preferences. For example, I always like my images to be +thumbnail'd. There's a handy web server built-in now, just run with `-s` +:metal: + +There is no need to install [Pillow] for handling images, that is now built-in. +The only external dependency is [exiftool] for image captions from embedded +metadata. If you don't have it `exiftool` installed, then it gracefully skips +that feature. So, you don't need to install anything else to run `distribusi` +now :pray: + +## Install + +``` +curl https://git.vvvvvvaria.org/decentral1se/distribusi-go/raw/branch/main/distribusi -o distribusi +chmod +x distribusi +./distribusi +``` + +## Hacking + +You'll need [Go] >= 1.13 installed. Run `go build .` to build a new `./distribusi` executable. + +[Go]: https://go.dev +[distribusi]: https://git.vvvvvvaria.org/varia/distribusi +[Pillow]: https://pillow.readthedocs.io/en/stable/installation.html#external-libraries +[exiftool]: https://exiftool.org/ +[report issues]: https://git.vvvvvvaria.org/decentral1se/distribusi-go/issues diff --git a/distribusi b/distribusi new file mode 100755 index 0000000..09850e9 Binary files /dev/null and b/distribusi differ diff --git a/distribusi.go b/distribusi.go new file mode 100644 index 0000000..9301d3b --- /dev/null +++ b/distribusi.go @@ -0,0 +1,640 @@ +// Package main is the command-line entrypoint for the distribusi command. +package main + +import ( + "bytes" + "encoding/base64" + "fmt" + "image/png" + "io/fs" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" + + logrusStack "github.com/Gurpartap/logrus-stack" + "github.com/barasher/go-exiftool" + "github.com/disintegration/imaging" + "github.com/gabriel-vasile/mimetype" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" +) + +// exiftooInstalled tells us if the exiftool binary is installed or not. +var exiftoolInstalled = true + +// generatedInDistribusi is an internal marker to help recognise when +// distribusi-go has generated files. +var generatedInDistribusi = "" + +// htmlBody is the template for the index.html files that are generated by distribusi-go. +var htmlBody = ` + + + + + %s + + + + + + + %s + +%s/", fname, fname) + div, err := mkDiv(c, mtype, href, fname, false) + if err != nil { + return err + } + + html = append(html, div) + } + + for _, file := range files { + fname := filepath.Base(file) + + if skip := shouldSkip(c, file, ignore); skip { + continue + } + + mtype, err := getMtype(file) + if err != nil { + return err + } + + unknown, href, err := getHref(c, file, mtype) + if err != nil { + return err + } + + div, err := mkDiv(c, mtype, href, fname, unknown) + if err != nil { + return err + } + + html = append(html, div) + } + + if err := writeIndex(absPath, html, c.String("css")); err != nil { + return err + } + + allDirs = append(allDirs, strings.Join(dirs, " ")) + } + + return nil + }); err != nil { + return err + } + + logrus.Debugf("generated files in following paths: %s", strings.Join(allDirs, " ")) + + return nil +} + +// removeIndex safely removes an index.html, making sure to check that +// distribusi generated it. +func removeIndex(fpath string) error { + file, err := os.ReadFile(fpath) + if err != nil { + return err + } + + if strings.Contains(string(file), generatedInDistribusi) { + if err := os.Remove(fpath); err != nil { + return err + } + + return nil + } + + return nil +} + +// getMtype determines file mimetype which helps to arrange HTML tags +// appropriate for the file type. +func getMtype(fpath string) (string, error) { + mtype, err := mimetype.DetectFile(fpath) + if err != nil { + return "", err + } + + return mtype.String(), nil +} + +// genThumb generates an in-memory image thumbnail and encodes it into a base64 +// representation which is suitable for embedding in HTML pages. +func genThumb(c *cli.Context, fpath, caption string) (string, error) { + knownFailureExts := []string{".ico", ".svg", ".xcf"} + if sliceContains(knownFailureExts, filepath.Ext(fpath)) { + return "", nil + } + + imgSrc, err := imaging.Open(fpath, imaging.AutoOrientation(true)) + if err != nil { + logrus.Debugf("failed to generate thumbnail for %s", fpath) + return "", err + } + + img := imaging.Thumbnail(imgSrc, 450, 450, imaging.Lanczos) + + buf := new(bytes.Buffer) + png.Encode(buf, img) + + imgBase64Str := base64.StdEncoding.EncodeToString(buf.Bytes()) + + return imgBase64Str, nil +} + +// getCaption retrieves an embedded image caption via exif-tool. If not +// exiftool is installed, we gracefully bail out. The caller is responsible for +// handling the alternative. +func getCaption(c *cli.Context, fpath string) (string, error) { + var caption string + + if !exiftoolInstalled { + return "", nil + } + + exif, err := exiftool.NewExiftool() + if err != nil { + return caption, fmt.Errorf("failed to initialise exiftool, saw %v", err) + } + defer exif.Close() + + for _, finfo := range exif.ExtractMetadata(fpath) { + if finfo.Err != nil { + continue + } + + for k, v := range finfo.Fields { + if k == "Comment" { + caption = fmt.Sprintf("%v", v) + } + } + } + + return caption, nil +} + +// parseMtype parses a mimetype string to simplify programmatic type lookups. +func parseMtype(mtype string) (string, string) { + stripCharset := strings.Split(mtype, ";") + splitTypes := strings.Split(stripCharset[0], "/") + + ftype, stype := splitTypes[0], splitTypes[1] + + return ftype, stype +} + +// sliceContains checks if an element is present in a list. +func sliceContains(items []string, target string) bool { + for _, item := range items { + if item == target { + return true + } + } + + return false +} + +// trimFinalNewline trims newlines from the end of bytes just read from files. +func trimFinalNewline(contents []byte) string { + return strings.TrimSuffix(string(contents), "\n") +} + +// getHref figures out which href tag corresponds to which file by navigating +// the mimetype. If a type of file is unknown, this is signalled via the bool +// return value. +func getHref(c *cli.Context, fpath string, mtype string) (bool, string, error) { + var href string + var caption string + var unknown bool + + fname := filepath.Base(fpath) + ftype, stype := parseMtype(mtype) + + if ftype == "text" { + fcontents, err := os.ReadFile(fpath) + if err != nil { + return unknown, href, err + } + if stype == "html" { + href = fmt.Sprintf("

", fname, trimFinalNewline(fcontents)) + } else { + href = fmt.Sprintf("

%s

", trimFinalNewline(fcontents)) + } + } else if ftype == "image" { + caption = "" + + exifCaption, err := getCaption(c, fpath) + if err != nil { + return unknown, href, nil + } + + if exifCaption != "" { + caption = exifCaption + } + + if stype == "gif" { + href = fmt.Sprintf("

", fname) + } else { + thumb, err := genThumb(c, fpath, caption) + if err != nil { + unknown = true + href = fmt.Sprintf("%s", stype, fname, fname) + } else { + href = fmt.Sprintf( + "

", + fname, thumb, caption, + ) + } + } + } else if ftype == "application" { + if stype == "pdf" { + href = fmt.Sprintf("

", fname, fname) + } else { + unknown = true + href = fmt.Sprintf("%s", stype, fname, fname) + } + } else if ftype == "audio" { + href = fmt.Sprintf("

", fname, stype) + } else if ftype == "video" { + href = fmt.Sprintf("

", fname, stype) + } else { + unknown = true + href = fmt.Sprintf("%s", stype, fname, fname) + } + + return unknown, href, nil +} + +// mkDiv cosntructs a HTML div for inclusion in the generated index.html. +func mkDiv(c *cli.Context, mtype string, href, fname string, unknown bool) (string, error) { + var div string + + filename := fmt.Sprintf("%s", fname) + + ftype, _ := parseMtype(mtype) + + if ftype == "text" { + div = fmt.Sprintf("

%s%s

", fname, ftype, href, filename) + } else if ftype == "os" { + div = fmt.Sprintf("

", fname, ftype, href) + } else { + if unknown { + // we really don't know what this is, so the filename is the href and we avoid adding it again + div = fmt.Sprintf("

", fname, ftype, href) + } else { + // images, videos, etc. still get a filename + div = fmt.Sprintf("

%s%s

", fname, ftype, href, filename) + } + } + + return div, nil +} + +// writeIndex writes a new index.html. +func writeIndex(fpath string, html []string, styles string) error { + body := fmt.Sprintf(htmlBody, generatedInDistribusi, "", strings.Join(html, "\n")) + HTMLPath := path.Join(fpath, "index.html") + contents := []byte(body) + + if styles != "" { + absPath, err := filepath.Abs(styles) + if err != nil { + return err + } + + if _, err := os.Stat(absPath); !os.IsNotExist(err) { + contents, err := os.ReadFile(absPath) + if err != nil { + return err + } + + body = fmt.Sprintf(htmlBody, generatedInDistribusi, contents, strings.Join(html, "\n")) + } + } + + if _, err := os.Stat(HTMLPath); err != nil { + if os.IsNotExist(err) { + if err := ioutil.WriteFile(HTMLPath, contents, 0644); err != nil { + return err + } + } else { + return err + } + } else { + file, err := os.ReadFile(HTMLPath) + if err != nil { + return err + } + + if strings.Contains(string(file), generatedInDistribusi) { + if err := ioutil.WriteFile(HTMLPath, contents, 0644); err != nil { + return err + } + } + } + + return nil +} + +// serveHTTP serves a web server for browsing distribusi. +func serveHTTP(fpath string) error { + fs := http.FileServer(http.Dir(fpath)) + http.Handle("/", fs) + + logrus.Info("distribusi live @ http://localhost:3000") + + if err := http.ListenAndServe(":3000", nil); err != nil { + return err + } + + return nil +} + +// wipeGeneratedFiles removes all distribusi generated files. +func wipeGeneratedFiles(dir string) error { + if err := filepath.WalkDir(dir, func(fpath string, dirEntry fs.DirEntry, err error) error { + fname := filepath.Base(fpath) + if fname == "index.html" { + if err := removeIndex(fpath); err != nil { + return err + } + } + return nil + }); err != nil { + return err + } + + return nil +} + +// shouldSkip checks if paths should be skipped over. +func shouldSkip(c *cli.Context, fpath string, ignore []string) bool { + if sliceContains(ignore, filepath.Base(fpath)) { + return true + } + + fpaths := strings.Split(fpath, "/") + for _, part := range fpaths { + if strings.HasPrefix(part, ".") { + if !c.Bool("hidden") { + return true + } + } + } + + return false +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..afa5810 --- /dev/null +++ b/go.mod @@ -0,0 +1,22 @@ +module varia.zone/distribusi + +go 1.17 + +require ( + github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4 + github.com/barasher/go-exiftool v1.7.0 + github.com/disintegration/imaging v1.6.2 + github.com/gabriel-vasile/mimetype v1.4.0 + github.com/sirupsen/logrus v1.8.1 + github.com/urfave/cli/v2 v2.3.0 +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect + golang.org/x/net v0.0.0-20210505024714-0287a6fb4125 // indirect + golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d83a5a9 --- /dev/null +++ b/go.sum @@ -0,0 +1,44 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4 h1:vdT7QwBhJJEVNFMBNhRSFDRCB6O16T28VhvqRgqFyn8= +github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4/go.mod h1:SvXOG8ElV28oAiG9zv91SDe5+9PfIr7PPccpr8YyXNs= +github.com/barasher/go-exiftool v1.7.0 h1:EOGb5D6TpWXmqsnEjJ0ai6+tIW2gZFwIoS9O/33Nixs= +github.com/barasher/go-exiftool v1.7.0/go.mod h1:F9s/a3uHSM8YniVfwF+sbQUtP8Gmh9nyzigNF+8vsWo= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= +github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/net v0.0.0-20210505024714-0287a6fb4125 h1:Ugb8sMTWuWRC3+sz5WeN/4kejDx9BvIwnPUiJBjJE+8= +golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=