distribusi-go/distribusi.go

// SPDX-License-Identifier: GPL-3.0-or-later

// Package main is the command-line entrypoint for the distribusi command.
package main

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"image/png"
	"io"
	"io/fs"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	logrusStack "github.com/Gurpartap/logrus-stack"
	"github.com/barasher/go-exiftool"
	"github.com/disintegration/imaging"
	"github.com/gabriel-vasile/mimetype"
	"github.com/k0kubun/go-ansi"
	"github.com/schollz/progressbar/v3"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli/v2"
)

// Version is the current version of distribusi-go.
var Version string

// Commit is the current commit of distribusi-go.
var Commit string

// port is the web server port.
var port = ":3000"

// exiftooInstalled tells us if the exiftool binary is installed or not.
var exiftoolInstalled = true

var logStripMsg = "...stripped from logs for brevity..."

// generatedInDistribusi is an internal marker to help recognise when
// distribusi-go has generated files.
var generatedInDistribusi = "<meta name='generator' content='distribusi-go' />"

// htmlBody is the template for the index.html files that are generated by distribusi-go.
var htmlBody = `
<!DOCTYPE html>
<html lang="en">
  <head>
    <!-- Generated with distribusi-go https://git.vvvvvvaria.org/decentral1se/distribusi-go -->
    %s

    <meta http-equiv="content-type" content="text/html; charset=utf-8">

    <style>
      .image {
        max-width: 100%%;
      }

      .pdf {
        width: 640px;
        height: 640px;
      }

      .x-directory::before {
        content: "📁 ";
        font-size: 18px;
      }

      .filename {
        display: block;
        font-family: mono;
      }

      .gif {
        width: 450px;
        max-height: 450px;
      }

      div {
        display: inline-block;
        vertical-align: top;
        margin: 1em;
        padding: 1em;
      }

      video {
        width: 450px;
        max-height: 450px;
      }

      %s
    </style>
  </head>
  <body>
    %s
  </body>
</html
`

// fileType is a file type identifier such as "image" or "audio".
type fileType = string

// subType is a more specific file type identifier compared to fileType, such as "gif" or "mp4".
type subType = string

// htmlTag is an appropriate HTML tag element for a specific fileType & subType.
type htmlTag = string

// htmlTags is a fileType/subType to htmlTag mapping.
var htmlTags = map[fileType]map[subType]htmlTag{
	"text": {
		"html":    `<section id="%s">%s</section>`,
		"generic": `<pre>%s</pre>`,
	},
	"image": {
		"thumbnail": trimAllNewlines(`<figure>
			<a href="%s">
				<img class="thumbnail" src="data:image/jpg;base64,%s">
			</a>
			<figcaption>%s</figcaption>
		</figure>`),
		"generic": trimAllNewlines(`<figure>
			<a href="%s">
				<img class=%s loading="lazy" src="%s"/>
			</a>
			<figcaption>%s</figcaption>
		</figure>`),
	},
	"application": {
		"pdf": trimAllNewlines(`<object data="%s" class="pdf" type="application/pdf">
			<embed src="%s" type="application/pdf"/>
		</object>`),
	},
	"audio": {
		"generic": trimAllNewlines(`<audio controls>
			<source src="%s" type="audio/%s">
			Your browser does not support the audio element.
		</audio>`),
	},
	"video": {
		"generic": trimAllNewlines(`<video controls>
			<source src="%s" type="video/%s">
		</video>`),
	},
	"unknown": {
		"generic": `<a class="%s" href="%s">%s</a>`,
	},
}

// main is the command-line entrypoint for the program.
func main() {
	if Version == "" {
		Version = "dev"
	}

	if Commit == "" {
		Commit = "       "
	}

	app := &cli.App{
		Name:    "distribusi-go",
		Version: fmt.Sprintf("%s-%s", Version, Commit[:7]),
		Usage:   "low-tech content management system for the web",
		Description: `
Distribusi is a content management system for the web that produces static
index pages based on folders in the files system. It is inspired by the
automatic index functions featured in several popular web servers. Distribusi
works by traversing the file system and directory hierarchy to automatically
list all the files in the directory, detect the file types and providing them
with relevant html classes and tags for easy styling.

Example:

    distribusi -p <path> -s
`,
		Flags: []cli.Flag{
			&cli.StringFlag{
				Name:    "css",
				Aliases: []string{"c"},
				Usage:   "css file for custom styles",
				Value:   "",
			},
			&cli.BoolFlag{
				Name:    "debug",
				Aliases: []string{"d"},
				Usage:   "debug logging",
				Value:   false,
			},
			&cli.BoolFlag{
				Name:    "hidden",
				Aliases: []string{"n"},
				Usage:   "include hidden directories (e.g. .git)",
				Value:   false,
			},
			&cli.StringFlag{
				Name:    "ignore",
				Aliases: []string{"i"},
				Usage:   "ignore specific paths, (e.g. \"*.gif, *.txt, mydir\" )",
				Value:   "",
			},
			&cli.StringFlag{
				Name:     "path",
				Aliases:  []string{"p"},
				Usage:    "path to distribusify",
				Required: true,
			},
			&cli.BoolFlag{
				Name:    "serve",
				Aliases: []string{"s"},
				Usage:   "serve distribusi for the web",
				Value:   false,
			},
			&cli.BoolFlag{
				Name:    "wipe",
				Aliases: []string{"w"},
				Usage:   "remove all generated files",
				Value:   false,
			},
		},
		Authors: []*cli.Author{
			{
				Name:  "Varia & friends",
				Email: "info@varia.zone",
			},
		},
		Before: func(c *cli.Context) error {
			if c.Bool("debug") {
				logrus.SetLevel(logrus.DebugLevel)
				logrus.SetFormatter(&logrus.TextFormatter{})

				logFile, err := getLogFile()
				if err != nil {
					logrus.Fatalf("unable to set up a log file, saw: %s", err)
				}

				logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
				logrus.RegisterExitHandler(func() {
					if logFile == nil {
						return
					}
					logFile.Close()
				})
				logrus.AddHook(logrusStack.StandardHook())
			}

			return nil
		},
		Action: func(c *cli.Context) error {
			root, err := filepath.Abs(c.String("path"))
			if err != nil {
				logrus.Fatal(err)
			}

			logrus.Debugf("selecting %s as distribusi root", root)

			var ignore []string
			if c.String("ignore") != "" {
				ignore = strings.Split(c.String("ignore"), ",")

				for i := range ignore {
					ignore[i] = strings.TrimSpace(ignore[i])
				}

				logrus.Debugf("parsed %s as ignore patterns", strings.Join(ignore, " "))
			}

			if c.Bool("wipe") {
				if err := wipeGeneratedFiles(root); err != nil {
					logrus.Fatal(err)
				}

				logrus.Infof("wiped all generated files in %s", root)

				return nil
			}

			_, err = exec.LookPath("exiftool")
			if err != nil {
				logrus.Debug("exiftool is not installed, skipping image captions")
				exiftoolInstalled = false
			}

			ch := make(chan error, 2)
			go func() {
				if err := distribusify(c, root, ignore); err != nil {
					ch <- err
					return
				}

				if c.Bool("serve") {
					fmt.Printf("done!")
				} else {
					fmt.Println("done!")
				}

				ch <- nil
				return
			}()

			if c.Bool("serve") {
				go func() {
					logrus.Debug("attempting to start up the web server")

					if err := serveHTTP(root); err != nil {
						ch <- err
						return
					}

					ch <- nil
					return
				}()
			} else {
				// close the channel, we're not serving anything
				ch <- nil
			}

			for i := 1; i <= 2; i++ {
				err := <-ch
				if err != nil {
					logrus.Fatal(err)
				}
			}

			return nil
		},
	}

	sort.Sort(cli.FlagsByName(app.Flags))
	sort.Sort(cli.CommandsByName(app.Commands))

	if err := app.Run(os.Args); err != nil {
		logrus.Fatal(err)
	}
}

// distribusify runs the main distribusi generation logic.
func distribusify(c *cli.Context, root string, ignore []string) error {
	var allDirs []string

	progress := mkProgressBar(c)

	if err := filepath.Walk(root, func(fpath string, finfo os.FileInfo, err error) error {

		skip, err := shouldSkip(c, fpath, ignore)
		if err != nil {
			return err
		}
		if skip {
			return nil
		}

		var html []string

		if finfo.IsDir() {
			var dirs []string
			var files []string

			absPath, err := filepath.Abs(fpath)
			if err != nil {
				logrus.Debugf("unable to read %s", absPath)
				return nil
			}

			contents, err := ioutil.ReadDir(absPath)
			if err != nil {
				logrus.Debugf("unable to read %s", absPath)
				return nil
			}

			for _, content := range contents {
				if content.IsDir() {
					dirs = append(dirs, path.Join(absPath, content.Name()))
				} else {
					if content.Name() == "index.html" {
						indexPath := path.Join(absPath, content.Name())
						file, err := os.ReadFile(indexPath)
						if err != nil {
							logrus.Debugf("unable to read %s, skipping", content.Name())
							continue
						}

						if strings.Contains(string(file), generatedInDistribusi) {
							logrus.Debugf("%s was not generated by distribusi, skipping", indexPath)
							continue
						}
					}

					files = append(files, path.Join(absPath, content.Name()))
				}
			}

			if len(dirs) == 0 && len(files) == 0 {
				return nil
			}

			if root != fpath {
				href := "<a href='../'>../</a>"
				div, err := mkDiv(c, "os/directory", href, "menu", false)
				if err != nil {
					return err
				}
				html = append(html, div)
			}

			for _, dir := range dirs {
				fname := filepath.Base(dir)

				skip, err := shouldSkip(c, fname, ignore)
				if err != nil {
					return err
				}
				if skip {
					continue
				}

				mtype := "os/directory"

				href := fmt.Sprintf("<a href='%s'>%s/</a>", fname, fname)
				div, err := mkDiv(c, mtype, href, fname, false)
				if err != nil {
					return err
				}

				html = append(html, div)
				progress.Add(1)
			}

			for _, file := range files {
				fname := filepath.Base(file)

				skip, err := shouldSkip(c, fname, ignore)
				if err != nil {
					return err
				}
				if skip {
					continue
				}

				mtype, err := getMtype(file)
				if err != nil {
					logrus.Debugf("failed to read mimetype of %s", file)
					continue
				}

				unknown, href, err := mkHref(c, file, mtype)
				if err != nil {
					logrus.Debugf("failed to generate href for %s", file)
					continue
				}

				div, err := mkDiv(c, mtype, href, fname, unknown)
				if err != nil {
					logrus.Debugf("failed to generate div for %s", file)
					continue
				}

				html = append(html, div)
				progress.Add(1)
			}

			if err := mkIndex(absPath, html, c.String("css")); err != nil {
				logrus.Debugf("unable to generated %s, skipping", path.Join(absPath, "index.html"))
				return nil
			}

			allDirs = append(allDirs, strings.Join(dirs, " "))
		}

		return nil
	}); err != nil {
		return err
	}

	logrus.Debugf("generated files in following paths: %s", strings.Join(allDirs, " "))

	return nil
}

// removeIndex safely removes an index.html, making sure to check that
// distribusi generated it.
func removeIndex(fpath string) error {
	file, err := os.ReadFile(fpath)
	if err != nil {
		return err
	}

	if strings.Contains(string(file), generatedInDistribusi) {
		if err := os.Remove(fpath); err != nil {
			return err
		}

		return nil
	}

	return nil
}

// getMtype reads file mimetypes directlry from files.
func getMtype(fpath string) (string, error) {
	mtype, err := mimetype.DetectFile(fpath)
	if err != nil {
		return "", err
	}

	return mtype.String(), nil
}

// genThumb generates an in-memory image thumbnail and encodes it into a base64
// representation which is suitable for embedding in HTML pages.
func genThumb(c *cli.Context, fpath, caption string) (string, error) {
	imgSrc, err := imaging.Open(fpath, imaging.AutoOrientation(true))
	if err != nil {
		return "", err
	}

	img := imaging.Thumbnail(imgSrc, 450, 450, imaging.Lanczos)

	buf := new(bytes.Buffer)
	png.Encode(buf, img)

	imgBase64Str := base64.StdEncoding.EncodeToString(buf.Bytes())

	logrus.Debugf("successfully generated thumbnail for %s", fpath)

	return imgBase64Str, nil
}

// getCaption retrieves an embedded image caption via exif-tool. If exiftool is
// not installed, we gracefully bail out. The caller is responsible for
// handling the alternative.
func getCaption(c *cli.Context, fpath string) (string, error) {
	var caption string

	if !exiftoolInstalled {
		return "", nil
	}

	exif, err := exiftool.NewExiftool()
	if err != nil {
		return caption, fmt.Errorf("failed to initialise exiftool, saw %v", err)
	}
	defer exif.Close()

	for _, finfo := range exif.ExtractMetadata(fpath) {
		if finfo.Err != nil {
			continue
		}

		for k, v := range finfo.Fields {
			if k == "Comment" {
				caption = fmt.Sprintf("%v", v)
			}
		}
	}

	if caption != "" {
		logrus.Debugf("retrieved caption %s from %s", caption, fpath)
	} else {
		logrus.Debugf("no comment retrieved for %s", fpath)
	}

	return caption, nil
}

// parseMtype parses a mimetype string to simplify programmatic type lookups.
func parseMtype(mtype string) (string, string, error) {
	if !strings.Contains(mtype, "/") {
		return "", "", fmt.Errorf("unable to parse %s", mtype)
	}

	stripCharset := strings.Split(mtype, ";")
	splitTypes := strings.Split(stripCharset[0], "/")
	ftype, stype := splitTypes[0], splitTypes[1]

	return ftype, stype, nil
}

// trimFinalNewline trims newlines from the end of bytes just read from files.
func trimFinalNewline(contents []byte) string {
	return strings.TrimSuffix(string(contents), "\n")
}

// trimAllNewlines removes all new lines.
func trimAllNewlines(contents string) string {
	return strings.ReplaceAll(string(contents), "\n", "")
}

// mkHref figures out which href tag corresponds to which file by navigating
// the mimetype. If a type of file is unknown, this is signalled via the bool
// return value.
func mkHref(c *cli.Context, fpath string, mtype string) (bool, string, error) {
	var href string
	var hrefTemplate string
	var strippedDebugOutput string
	var unknown bool

	fname := filepath.Base(fpath)
	ftype, stype, err := parseMtype(mtype)
	if err != nil {
		return unknown, href, err
	}

	if ftype == "text" {
		fcontents, err := os.ReadFile(fpath)
		if err != nil {
			return unknown, href, err
		}

		trimmed := trimFinalNewline(fcontents)

		if stype == "html" {
			hrefTemplate = htmlTags[ftype][stype]
			href = fmt.Sprintf(hrefTemplate, fname, trimmed)
			strippedDebugOutput = fmt.Sprintf(hrefTemplate, fname, logStripMsg)
		} else {
			hrefTemplate = htmlTags[ftype]["generic"]
			href = fmt.Sprintf(hrefTemplate, trimmed)
			strippedDebugOutput = fmt.Sprintf(hrefTemplate, logStripMsg)
		}
	} else if ftype == "image" {
		caption, err := getCaption(c, fpath)
		if err != nil {
			return unknown, href, nil
		}

		if stype == "gif" {
			hrefTemplate = htmlTags[ftype]["generic"]
			href = fmt.Sprintf(hrefTemplate, fname, stype, fname, caption)
		} else {
			thumb, err := genThumb(c, fpath, caption)
			if err != nil {
				hrefTemplate = htmlTags[ftype]["generic"]
				href = fmt.Sprintf(hrefTemplate, fname, stype, fname, caption)
				logrus.Debugf("failed to generate thumbnail for %s, showing original image", fpath)
			} else {
				hrefTemplate = htmlTags[ftype]["thumbnail"]
				href = fmt.Sprintf(hrefTemplate, fname, thumb, caption)
				strippedDebugOutput = fmt.Sprintf(hrefTemplate, fname, logStripMsg, caption)
			}
		}
	} else if ftype == "application" {
		if stype == "pdf" {
			hrefTemplate = htmlTags[ftype][stype]
			href = fmt.Sprintf(hrefTemplate, fname, fname)
		} else {
			unknown = true
			hrefTemplate = htmlTags["unknown"]["generic"]
			href = fmt.Sprintf(hrefTemplate, stype, fname, fname)
		}
	} else if ftype == "audio" {
		hrefTemplate = htmlTags[ftype]["generic"]
		href = fmt.Sprintf(hrefTemplate, fname, stype)
	} else if ftype == "video" {
		hrefTemplate = htmlTags[ftype]["generic"]
		href = fmt.Sprintf(hrefTemplate, fname, stype)
	} else {
		unknown = true
		hrefTemplate = htmlTags["unknown"]["generic"]
		href = fmt.Sprintf(hrefTemplate, stype, fname, fname)
	}

	if strippedDebugOutput != "" {
		logrus.Debugf("%s was wrapped in: %s", fname, strippedDebugOutput)
	} else {
		logrus.Debugf("%s was wrapped in: %s", fname, href)
	}

	return unknown, href, nil
}

// mkDiv cosntructs a HTML div for inclusion in the generated index.html. These
// dives are used to wrap the elements that appear on generated pages with
// relevant identifiers for convenient styling.
func mkDiv(c *cli.Context, mtype string, href, fname string, unknown bool) (string, error) {
	var div string
	var divTemplate string
	var strippedDebugOutput string

	filename := fmt.Sprintf("<span class='filename'>%s</span>", fname)

	ftype, stype, err := parseMtype(mtype)
	if err != nil {
		return div, err
	}

	if ftype == "text" {
		divTemplate = "<div id=\"%s\" class='%s'>%s%s</div>"
		div = fmt.Sprintf(divTemplate, fname, ftype, href, filename)
		strippedDebugOutput = fmt.Sprintf(divTemplate, fname, ftype, logStripMsg, filename)
	} else if ftype == "os" {
		if stype == "directory" {
			divTemplate = "<div id=\"%s\" class='x-%s'>%s</div>"
			div = fmt.Sprintf(divTemplate, fname, stype, href)
		} else {
			// don't include filename since link already has it
			divTemplate = "<div id=\"%s\" class='%s'>%s</div>"
			div = fmt.Sprintf(divTemplate, fname, ftype, href)
		}
	} else {
		if unknown {
			// don't include filename since link already has it
			divTemplate = "<div id=\"%s\" class='%s'>%s</div>"
			div = fmt.Sprintf(divTemplate, fname, ftype, href)
		} else {
			divTemplate = "<div id=\"%s\" class='%s'>%s%s</div>"
			div = fmt.Sprintf(divTemplate, fname, ftype, href, filename)
			strippedDebugOutput = fmt.Sprintf(divTemplate, fname, ftype, logStripMsg, filename)
		}
	}

	if strippedDebugOutput != "" {
		logrus.Debugf("%s was wrapped in: %s", fname, strippedDebugOutput)
	} else {
		logrus.Debugf("%s was wrapped in: %s", fname, div)
	}

	return div, nil
}

// mkIndex writes a new generated index.html file to the file system.
func mkIndex(fpath string, html []string, styles string) error {
	body := fmt.Sprintf(htmlBody, generatedInDistribusi, "", strings.Join(html, "\n"))

	if styles != "" {
		absPath, err := filepath.Abs(styles)
		if err != nil {
			return err
		}

		if _, err := os.Stat(absPath); !os.IsNotExist(err) {
			contents, err := os.ReadFile(absPath)
			if err != nil {
				return nil
			}

			logrus.Debugf("loading custom styles from %s", absPath)

			body = fmt.Sprintf(htmlBody, generatedInDistribusi, contents, strings.Join(html, "\n"))
		}
	}

	HTMLPath := path.Join(fpath, "index.html")
	contents := []byte(body)

	if _, err := os.Stat(HTMLPath); err != nil {
		if os.IsNotExist(err) {
			if err := ioutil.WriteFile(HTMLPath, contents, 0644); err != nil {
				logrus.Debugf("unable to write %s, skipping", HTMLPath)
				return nil
			}
		} else {
			logrus.Debugf("unable to read %s, skipping", HTMLPath)
			return nil
		}
	} else {
		file, err := os.ReadFile(HTMLPath)
		if err != nil {
			logrus.Debugf("unable to read %s, skipping", HTMLPath)
			return nil
		}

		if strings.Contains(string(file), generatedInDistribusi) {
			if err := ioutil.WriteFile(HTMLPath, contents, 0644); err != nil {
				logrus.Debugf("unable to write %s, skipping", HTMLPath)
				return nil
			}
		}
	}

	return nil
}

// serveHTTP serves a web server for browsing distribusi generated files. This
// is mostly convenient when doing development work or showing something
// quickly on your work station. It should be fine to serve "for production"
// though too as it uses the stdlib Go HTTP server. Distribusi still works just
// fine with the usual Nginx, Apache, etc.
func serveHTTP(fpath string) error {
	fs := http.FileServer(http.Dir(fpath))
	http.Handle("/", fs)

	if err := http.ListenAndServe(port, nil); err != nil {
		return err
	}

	return nil
}

// wipeGeneratedFiles removes all distribusi generated files under a file
// system path. We do take care to avoid deleting files that distribusi has not
// generated by checking their contents.
func wipeGeneratedFiles(dir string) error {
	if err := filepath.WalkDir(dir, func(fpath string, dirEntry fs.DirEntry, err error) error {
		fname := filepath.Base(fpath)
		if fname == "index.html" {
			if err := removeIndex(fpath); err != nil {
				return err
			}
			logrus.Debugf("wiping %s as requested", fpath)
		}
		return nil
	}); err != nil {
		return err
	}

	return nil
}

// shouldSkip checks if a specific file system path should be skipped over when
// running distribusi file generation. This might happen due to being a hidden
// directory or matching a pattern provided by the end-user.
func shouldSkip(c *cli.Context, fpath string, ignore []string) (bool, error) {
	for _, pattern := range ignore {
		match, err := filepath.Match(pattern, filepath.Base(fpath))
		if err != nil {
			return false, err
		}
		if match {
			logrus.Debugf("skipping %s, matched %s", filepath.Base(fpath), pattern)
			return true, nil
		}
	}

	fpaths := strings.Split(fpath, "/")
	for _, part := range fpaths {
		if strings.HasPrefix(part, ".") {
			if !c.Bool("hidden") {
				logrus.Debugf("skipping %s, hidden directory", fpath)
				return true, nil
			}
		}
	}

	return false, nil
}

// getLogFile creates a new log file for debug output. We do this because the
// standard debug listing is quite verbose and it is often more convenient to
// read it from file. Also handier for bug reports.
func getLogFile() (*os.File, error) {
	cTime := time.Now()
	timeNow := fmt.Sprintf("%v-%v-%v", cTime.Hour(), cTime.Minute(), cTime.Second())
	prefix := fmt.Sprintf("distribusi-go-%s", timeNow)

	file, err := ioutil.TempFile("/tmp", prefix)
	if err != nil {
		return nil, err
	}

	logrus.Debugf("creating %s as debug log file", file.Name())

	return file, nil
}

// mkProgressBar creates a customised progress bar. This bar is used to give
// real-time updates on the progress of running distribusi.
func mkProgressBar(c *cli.Context) *progressbar.ProgressBar {
	var description string
	if c.Bool("serve") {
		description = fmt.Sprintf("distribusifying... live @ http://localhost%s", port)
	} else {
		description = "distribusifying..."
	}

	bar := progressbar.NewOptions(-1,
		progressbar.OptionSetDescription(description),
		progressbar.OptionSetWriter(ansi.NewAnsiStdout()),
		progressbar.OptionEnableColorCodes(true),
		progressbar.OptionShowCount(),
		progressbar.OptionSpinnerType(9),
	)

	return bar
}