// SPDX-License-Identifier: GPL-3.0-or-later
// Package main is the command-line entrypoint for the distribusi command.
package main
import (
"bytes"
"encoding/base64"
"fmt"
"image/png"
"io"
"io/fs"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strings"
"time"
logrusStack "github.com/Gurpartap/logrus-stack"
"github.com/barasher/go-exiftool"
"github.com/disintegration/imaging"
"github.com/gabriel-vasile/mimetype"
"github.com/k0kubun/go-ansi"
"github.com/schollz/progressbar/v3"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
)
// Version is the current version of distribusi-go
var Version string
// Commit is the current commit of distribusi-go
var Commit string
// port is the web server port
var port = ":3000"
// exiftooInstalled tells us if the exiftool binary is installed or not.
var exiftoolInstalled = true
// generatedInDistribusi is an internal marker to help recognise when
// distribusi-go has generated files.
var generatedInDistribusi = ""
// htmlBody is the template for the index.html files that are generated by distribusi-go.
var htmlBody = `
%s
%s
-s
`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "css",
Aliases: []string{"c"},
Usage: "css file for custom styles",
Value: "",
},
&cli.BoolFlag{
Name: "debug",
Aliases: []string{"d"},
Usage: "debug logging",
Value: false,
},
&cli.BoolFlag{
Name: "hidden",
Aliases: []string{"n"},
Usage: "include hidden directories (e.g. .git)",
Value: false,
},
&cli.StringFlag{
Name: "ignore",
Aliases: []string{"i"},
Usage: "ignore specific paths, (e.g. \"*.gif, *.txt, mydir\" )",
Value: "",
},
&cli.StringFlag{
Name: "path",
Aliases: []string{"p"},
Usage: "path to distribusify",
Required: true,
},
&cli.BoolFlag{
Name: "serve",
Aliases: []string{"s"},
Usage: "serve distribusi for the web",
Value: false,
},
&cli.BoolFlag{
Name: "wipe",
Aliases: []string{"w"},
Usage: "remove all generated files",
Value: false,
},
},
Authors: []*cli.Author{
{
Name: "Varia & friends",
Email: "info@varia.zone",
},
},
Before: func(c *cli.Context) error {
if c.Bool("debug") {
logrus.SetLevel(logrus.DebugLevel)
logrus.SetFormatter(&logrus.TextFormatter{})
logFile, err := getLogFile()
if err != nil {
logrus.Fatalf("unable to set up a log file, saw: %s", err)
}
logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
logrus.RegisterExitHandler(func() {
if logFile == nil {
return
}
logFile.Close()
})
logrus.AddHook(logrusStack.StandardHook())
}
return nil
},
Action: func(c *cli.Context) error {
root, err := filepath.Abs(c.String("path"))
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("selecting %s as distribusi root", root)
var ignore []string
if c.String("ignore") != "" {
ignore = strings.Split(c.String("ignore"), ",")
for i := range ignore {
ignore[i] = strings.TrimSpace(ignore[i])
}
logrus.Debugf("parsed %s as ignore patterns", strings.Join(ignore, " "))
}
if c.Bool("wipe") {
if err := wipeGeneratedFiles(root); err != nil {
logrus.Fatal(err)
}
logrus.Infof("wiped all generated files in %s", root)
return nil
}
_, err = exec.LookPath("exiftool")
if err != nil {
logrus.Debug("exiftool is not installed, skipping image captions")
exiftoolInstalled = false
}
ch := make(chan error, 2)
go func() {
if err := distribusify(c, root, ignore); err != nil {
ch <- err
return
}
fmt.Printf("done!")
ch <- nil
return
}()
if c.Bool("serve") {
go func() {
logrus.Debug("starting up web server")
if err := serveHTTP(root); err != nil {
ch <- err
return
}
ch <- nil
return
}()
}
for i := 1; i <= 2; i++ {
err := <-ch
if err != nil {
logrus.Fatal(err)
}
}
return nil
},
}
sort.Sort(cli.FlagsByName(app.Flags))
sort.Sort(cli.CommandsByName(app.Commands))
if err := app.Run(os.Args); err != nil {
logrus.Fatal(err)
}
}
// distribusify runs the main distribusi generation logic.
func distribusify(c *cli.Context, root string, ignore []string) error {
var allDirs []string
progress := mkProgressBar()
if err := filepath.Walk(root, func(fpath string, finfo os.FileInfo, err error) error {
skip, err := shouldSkip(c, fpath, ignore)
if err != nil {
return err
}
if skip {
return nil
}
var html []string
if finfo.IsDir() {
var dirs []string
var files []string
absPath, err := filepath.Abs(fpath)
if err != nil {
logrus.Debugf("unable to read %s", absPath)
return nil
}
contents, err := ioutil.ReadDir(absPath)
if err != nil {
logrus.Debugf("unable to read %s", absPath)
return nil
}
for _, content := range contents {
if content.IsDir() {
dirs = append(dirs, path.Join(absPath, content.Name()))
} else {
if content.Name() == "index.html" {
indexPath := path.Join(absPath, content.Name())
file, err := os.ReadFile(indexPath)
if err != nil {
logrus.Debugf("unable to read %s, skipping", content.Name())
continue
}
if strings.Contains(string(file), generatedInDistribusi) {
logrus.Debugf("%s was not generated by distribusi, skipping", indexPath)
continue
}
}
files = append(files, path.Join(absPath, content.Name()))
}
}
if len(dirs) == 0 && len(files) == 0 {
return nil
}
if root != fpath {
href := "../"
div, err := mkDiv(c, "os/directory", href, "menu", false)
if err != nil {
return err
}
html = append(html, div)
}
for _, dir := range dirs {
fname := filepath.Base(dir)
skip, err := shouldSkip(c, fname, ignore)
if err != nil {
return err
}
if skip {
continue
}
mtype := "os/directory"
href := fmt.Sprintf("%s/", fname, fname)
div, err := mkDiv(c, mtype, href, fname, false)
if err != nil {
return err
}
html = append(html, div)
progress.Add(1)
}
for _, file := range files {
fname := filepath.Base(file)
skip, err := shouldSkip(c, fname, ignore)
if err != nil {
return err
}
if skip {
continue
}
mtype, err := getMtype(file)
if err != nil {
logrus.Debugf("failed to read mimetype of %s", file)
continue
}
unknown, href, err := getHref(c, file, mtype)
if err != nil {
logrus.Debugf("failed to generate href for %s", file)
continue
}
div, err := mkDiv(c, mtype, href, fname, unknown)
if err != nil {
logrus.Debugf("failed to generate div for %s", file)
continue
}
html = append(html, div)
progress.Add(1)
}
if err := writeIndex(absPath, html, c.String("css")); err != nil {
logrus.Debugf("unable to generated %s, skipping", path.Join(absPath, "index.html"))
return nil
}
allDirs = append(allDirs, strings.Join(dirs, " "))
}
return nil
}); err != nil {
return err
}
logrus.Debugf("generated files in following paths: %s", strings.Join(allDirs, " "))
return nil
}
// removeIndex safely removes an index.html, making sure to check that
// distribusi generated it.
func removeIndex(fpath string) error {
file, err := os.ReadFile(fpath)
if err != nil {
return err
}
if strings.Contains(string(file), generatedInDistribusi) {
if err := os.Remove(fpath); err != nil {
return err
}
return nil
}
return nil
}
// getMtype determines file mimetype which helps to arrange HTML tags
// appropriate for the file type.
func getMtype(fpath string) (string, error) {
mtype, err := mimetype.DetectFile(fpath)
if err != nil {
return "", err
}
return mtype.String(), nil
}
// genThumb generates an in-memory image thumbnail and encodes it into a base64
// representation which is suitable for embedding in HTML pages.
func genThumb(c *cli.Context, fpath, caption string) (string, error) {
imgSrc, err := imaging.Open(fpath, imaging.AutoOrientation(true))
if err != nil {
return "", err
}
img := imaging.Thumbnail(imgSrc, 450, 450, imaging.Lanczos)
buf := new(bytes.Buffer)
png.Encode(buf, img)
imgBase64Str := base64.StdEncoding.EncodeToString(buf.Bytes())
logrus.Debugf("successfully generated thumbnail for %s", fpath)
return imgBase64Str, nil
}
// getCaption retrieves an embedded image caption via exif-tool. If not
// exiftool is installed, we gracefully bail out. The caller is responsible for
// handling the alternative.
func getCaption(c *cli.Context, fpath string) (string, error) {
var caption string
if !exiftoolInstalled {
return "", nil
}
exif, err := exiftool.NewExiftool()
if err != nil {
return caption, fmt.Errorf("failed to initialise exiftool, saw %v", err)
}
defer exif.Close()
for _, finfo := range exif.ExtractMetadata(fpath) {
if finfo.Err != nil {
continue
}
for k, v := range finfo.Fields {
if k == "Comment" {
caption = fmt.Sprintf("%v", v)
}
}
}
if caption != "" {
logrus.Debugf("retrieved caption %s from %s", caption, fpath)
} else {
logrus.Debugf("no comment retrieved for %s", fpath)
}
return caption, nil
}
// parseMtype parses a mimetype string to simplify programmatic type lookups.
func parseMtype(mtype string) (string, string) {
stripCharset := strings.Split(mtype, ";")
splitTypes := strings.Split(stripCharset[0], "/")
ftype, stype := splitTypes[0], splitTypes[1]
return ftype, stype
}
// sliceContains checks if an element is present in a list.
func sliceContains(items []string, target string) bool {
for _, item := range items {
if item == target {
return true
}
}
return false
}
// trimFinalNewline trims newlines from the end of bytes just read from files.
func trimFinalNewline(contents []byte) string {
return strings.TrimSuffix(string(contents), "\n")
}
// getHref figures out which href tag corresponds to which file by navigating
// the mimetype. If a type of file is unknown, this is signalled via the bool
// return value.
func getHref(c *cli.Context, fpath string, mtype string) (bool, string, error) {
var href string
var unknown bool
fname := filepath.Base(fpath)
ftype, stype := parseMtype(mtype)
if ftype == "text" {
fcontents, err := os.ReadFile(fpath)
if err != nil {
return unknown, href, err
}
if stype == "html" {
href = fmt.Sprintf("%s", fname, trimFinalNewline(fcontents))
} else {
href = fmt.Sprintf("
%s
", trimFinalNewline(fcontents))
}
} else if ftype == "image" {
var caption string
exifCaption, err := getCaption(c, fpath)
if err != nil {
return unknown, href, nil
}
if exifCaption != "" {
caption = exifCaption
}
if stype == "gif" {
href = fmt.Sprintf("", fname)
} else {
thumb, err := genThumb(c, fpath, caption)
if err != nil {
logrus.Debugf("failed to generate thumbnail for %s, showing original image", fpath)
href = fmt.Sprintf("", stype, fname)
} else {
href = fmt.Sprintf(
"%s",
fname, thumb, caption,
)
}
}
} else if ftype == "application" {
if stype == "pdf" {
href = fmt.Sprintf("", fname, fname)
} else {
unknown = true
href = fmt.Sprintf("%s", stype, fname, fname)
}
} else if ftype == "audio" {
href = fmt.Sprintf("", fname, stype)
} else if ftype == "video" {
href = fmt.Sprintf("", fname, stype)
} else {
unknown = true
href = fmt.Sprintf("%s", stype, fname, fname)
}
logrus.Debugf("generated href for %s: %s", fname, href)
return unknown, href, nil
}
// mkDiv cosntructs a HTML div for inclusion in the generated index.html.
func mkDiv(c *cli.Context, mtype string, href, fname string, unknown bool) (string, error) {
var div string
filename := fmt.Sprintf("%s", fname)
ftype, _ := parseMtype(mtype)
if ftype == "text" {
div = fmt.Sprintf("
%s%s
", fname, ftype, href, filename)
} else if ftype == "os" {
div = fmt.Sprintf("
%s
", fname, ftype, href)
} else {
if unknown {
// we really don't know what this is, so the filename is the href and we avoid adding it again
div = fmt.Sprintf("
%s
", fname, ftype, href)
} else {
// images, videos, etc. still get a filename
div = fmt.Sprintf("
%s%s
", fname, ftype, href, filename)
}
}
logrus.Debugf("generated div wrapper for %s: %s", fname, div)
return div, nil
}
// writeIndex writes a new index.html.
func writeIndex(fpath string, html []string, styles string) error {
body := fmt.Sprintf(htmlBody, generatedInDistribusi, "", strings.Join(html, "\n"))
if styles != "" {
absPath, err := filepath.Abs(styles)
if err != nil {
return err
}
if _, err := os.Stat(absPath); !os.IsNotExist(err) {
contents, err := os.ReadFile(absPath)
if err != nil {
return nil
}
logrus.Debugf("loading custom styles from %s", absPath)
body = fmt.Sprintf(htmlBody, generatedInDistribusi, contents, strings.Join(html, "\n"))
}
}
HTMLPath := path.Join(fpath, "index.html")
contents := []byte(body)
if _, err := os.Stat(HTMLPath); err != nil {
if os.IsNotExist(err) {
if err := ioutil.WriteFile(HTMLPath, contents, 0644); err != nil {
logrus.Debugf("unable to write %s, skipping", HTMLPath)
return nil
}
} else {
logrus.Debugf("unable to read %s, skipping", HTMLPath)
return nil
}
} else {
file, err := os.ReadFile(HTMLPath)
if err != nil {
logrus.Debugf("unable to read %s, skipping", HTMLPath)
return nil
}
if strings.Contains(string(file), generatedInDistribusi) {
if err := ioutil.WriteFile(HTMLPath, contents, 0644); err != nil {
logrus.Debugf("unable to write %s, skipping", HTMLPath)
return nil
}
}
}
return nil
}
// serveHTTP serves a web server for browsing distribusi.
func serveHTTP(fpath string) error {
fs := http.FileServer(http.Dir(fpath))
http.Handle("/", fs)
logrus.Infof("distribusi serving from http://localhost%s", port)
if err := http.ListenAndServe(port, nil); err != nil {
return err
}
return nil
}
// wipeGeneratedFiles removes all distribusi generated files.
func wipeGeneratedFiles(dir string) error {
if err := filepath.WalkDir(dir, func(fpath string, dirEntry fs.DirEntry, err error) error {
fname := filepath.Base(fpath)
if fname == "index.html" {
if err := removeIndex(fpath); err != nil {
return err
}
logrus.Debugf("wiping %s as requested", fpath)
}
return nil
}); err != nil {
return err
}
return nil
}
// shouldSkip checks if paths should be skipped over.
func shouldSkip(c *cli.Context, fpath string, ignore []string) (bool, error) {
for _, pattern := range ignore {
match, err := filepath.Match(pattern, filepath.Base(fpath))
if err != nil {
return false, err
}
if match {
logrus.Debugf("skipping %s, matched %s", filepath.Base(fpath), pattern)
return true, nil
}
}
fpaths := strings.Split(fpath, "/")
for _, part := range fpaths {
if strings.HasPrefix(part, ".") {
if !c.Bool("hidden") {
logrus.Debugf("skipping %s, hidden directory", fpath)
return true, nil
}
}
}
return false, nil
}
// getLogFile creates a new log file for debug output.
func getLogFile() (*os.File, error) {
cTime := time.Now()
timeNow := fmt.Sprintf("%v-%v-%v", cTime.Hour(), cTime.Minute(), cTime.Second())
prefix := fmt.Sprintf("distribusi-go-%s", timeNow)
file, err := ioutil.TempFile("/tmp", prefix)
if err != nil {
return nil, err
}
logrus.Debugf("creating %s as debug log file", file.Name())
return file, nil
}
// mkProgressBar creates a progress bar.
func mkProgressBar() *progressbar.ProgressBar {
bar := progressbar.NewOptions(-1,
progressbar.OptionSetDescription("distribusifying..."),
progressbar.OptionSetWriter(ansi.NewAnsiStdout()),
progressbar.OptionEnableColorCodes(true),
progressbar.OptionShowCount(),
progressbar.OptionSpinnerType(9),
)
return bar
}
