A low-tech content management system for the web
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

857 lines
20 KiB

// SPDX-License-Identifier: GPL-3.0-or-later
3 years ago
// Package main is the command-line entrypoint for the distribusi command.
package main
import (
"bytes"
"encoding/base64"
"fmt"
"image/png"
"io"
3 years ago
"io/fs"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strings"
"time"
3 years ago
logrusStack "github.com/Gurpartap/logrus-stack"
"github.com/barasher/go-exiftool"
"github.com/disintegration/imaging"
"github.com/gabriel-vasile/mimetype"
"github.com/k0kubun/go-ansi"
"github.com/schollz/progressbar/v3"
3 years ago
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
)
// Version is the current version of distribusi-go
var Version string
// Commit is the current commit of distribusi-go
var Commit string
// port is the web server port
var port = ":3000"
3 years ago
// exiftooInstalled tells us if the exiftool binary is installed or not.
var exiftoolInstalled = true
var logStripMsg = "...stripped from logs for brevity..."
3 years ago
// generatedInDistribusi is an internal marker to help recognise when
// distribusi-go has generated files.
var generatedInDistribusi = "<meta name='generator' content='distribusi-go' />"
// htmlBody is the template for the index.html files that are generated by distribusi-go.
var htmlBody = `
<!DOCTYPE html>
<html lang="en">
<head>
<!-- Generated with distribusi-go https://git.vvvvvvaria.org/decentral1se/distribusi-go -->
%s
<meta http-equiv="content-type" content="text/html; charset=utf-8">
<style>
.image {
max-width: 100%%;
}
.pdf {
width: 640px;
height: 640px;
}
.x-directory::before {
3 years ago
content: "📁 ";
font-size: 18px;
}
.filename {
display: block;
font-family: mono;
}
.gif {
width: 450px;
max-height: 450px;
}
div {
display: inline-block;
vertical-align: top;
margin: 1em;
padding: 1em;
}
video {
width: 450px;
max-height: 450px;
}
%s
</style>
</head>
<body>
3 years ago
%s
</body>
3 years ago
</html
`
// fileType is a file type identifier such as "image" or "audio".
type fileType = string
// subType is a more specific file type identifier compared to fileType, such as "gif" and "mp4".
type subType = string
// htmlTag is an appropriate html tag element for a specific fileType & subType.
type htmlTag = string
// htmlTags is a fileType/subType to htmlTag mapping for HTML generation purposes.
var htmlTags = map[fileType]map[subType]htmlTag{
"text": {
"html": `<section id="%s">%s</section>`,
"generic": `<pre>%s</pre>`,
},
"image": {
"thumbnail": trimAllNewlines(`<figure>
<a href="%s">
<img class="thumbnail" src="data:image/jpg;base64,%s">
</a>
<figcaption>%s</figcaption>
</figure>`),
"generic": trimAllNewlines(`<figure>
<a href="%s">
<img class=%s loading="lazy" src="%s"/>
</a>
<figcaption>%s</figcaption>
</figure>`),
},
"application": {
"pdf": trimAllNewlines(`<object data="%s" class="pdf" type="application/pdf">
<embed src="%s" type="application/pdf"/>
</object>`),
},
"audio": {
"generic": trimAllNewlines(`<audio controls>
<source src="%s" type="audio/%s">
Your browser does not support the audio element.
</audio>`),
},
"video": {
"generic": trimAllNewlines(`<video controls>
<source src="%s" type="video/%s">
</video>`),
},
"unknown": {
"generic": `<a class="%s" href="%s">%s</a>`,
},
}
3 years ago
// main is the command-line entrypoint.
func main() {
if Version == "" {
Version = "dev"
}
if Commit == "" {
Commit = " "
}
3 years ago
app := &cli.App{
Name: "distribusi-go",
Version: fmt.Sprintf("%s-%s", Version, Commit[:7]),
3 years ago
Usage: "low-tech content management system for the web",
Description: `
Distribusi is a content management system for the web that produces static
index pages based on folders in the files system. It is inspired by the
automatic index functions featured in several popular web servers. Distribusi
works by traversing the file system and directory hierarchy to automatically
list all the files in the directory, detect the file types and providing them
with relevant html classes and tags for easy styling.
Example:
distribusi -p <path> -s
3 years ago
`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "css",
Aliases: []string{"c"},
Usage: "css file for custom styles",
Value: "",
},
&cli.BoolFlag{
Name: "debug",
Aliases: []string{"d"},
Usage: "debug logging",
Value: false,
},
&cli.BoolFlag{
Name: "hidden",
Aliases: []string{"n"},
Usage: "include hidden directories (e.g. .git)",
3 years ago
Value: false,
},
&cli.StringFlag{
Name: "ignore",
Aliases: []string{"i"},
Usage: "ignore specific paths, (e.g. \"*.gif, *.txt, mydir\" )",
3 years ago
Value: "",
},
&cli.StringFlag{
Name: "path",
Aliases: []string{"p"},
Usage: "path to distribusify",
Required: true,
},
&cli.BoolFlag{
Name: "serve",
Aliases: []string{"s"},
Usage: "serve distribusi for the web",
Value: false,
},
&cli.BoolFlag{
Name: "wipe",
Aliases: []string{"w"},
Usage: "remove all generated files",
Value: false,
},
},
Authors: []*cli.Author{
{
Name: "Varia & friends",
Email: "info@varia.zone",
3 years ago
},
},
Before: func(c *cli.Context) error {
if c.Bool("debug") {
logrus.SetLevel(logrus.DebugLevel)
logrus.SetFormatter(&logrus.TextFormatter{})
logFile, err := getLogFile()
if err != nil {
logrus.Fatalf("unable to set up a log file, saw: %s", err)
}
logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
logrus.RegisterExitHandler(func() {
if logFile == nil {
return
}
logFile.Close()
})
3 years ago
logrus.AddHook(logrusStack.StandardHook())
}
return nil
},
Action: func(c *cli.Context) error {
root, err := filepath.Abs(c.String("path"))
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("selecting %s as distribusi root", root)
3 years ago
var ignore []string
if c.String("ignore") != "" {
ignore = strings.Split(c.String("ignore"), ",")
for i := range ignore {
ignore[i] = strings.TrimSpace(ignore[i])
}
logrus.Debugf("parsed %s as ignore patterns", strings.Join(ignore, " "))
}
if c.Bool("wipe") {
if err := wipeGeneratedFiles(root); err != nil {
logrus.Fatal(err)
}
logrus.Infof("wiped all generated files in %s", root)
return nil
}
_, err = exec.LookPath("exiftool")
if err != nil {
logrus.Debug("exiftool is not installed, skipping image captions")
3 years ago
exiftoolInstalled = false
}
ch := make(chan error, 2)
go func() {
if err := distribusify(c, root, ignore); err != nil {
ch <- err
return
}
fmt.Printf("done!")
ch <- nil
return
}()
3 years ago
if c.Bool("serve") {
go func() {
logrus.Debug("attempting to start up the web server")
if err := serveHTTP(root); err != nil {
ch <- err
return
}
ch <- nil
return
}()
}
for i := 1; i <= 2; i++ {
err := <-ch
if err != nil {
3 years ago
logrus.Fatal(err)
}
}
return nil
},
}
sort.Sort(cli.FlagsByName(app.Flags))
sort.Sort(cli.CommandsByName(app.Commands))
if err := app.Run(os.Args); err != nil {
logrus.Fatal(err)
}
}
// distribusify runs the main distribusi generation logic.
func distribusify(c *cli.Context, root string, ignore []string) error {
var allDirs []string
progress := mkProgressBar()
3 years ago
if err := filepath.Walk(root, func(fpath string, finfo os.FileInfo, err error) error {
skip, err := shouldSkip(c, fpath, ignore)
if err != nil {
return err
}
if skip {
return nil
3 years ago
}
var html []string
if finfo.IsDir() {
var dirs []string
var files []string
absPath, err := filepath.Abs(fpath)
if err != nil {
logrus.Debugf("unable to read %s", absPath)
return nil
3 years ago
}
contents, err := ioutil.ReadDir(absPath)
if err != nil {
logrus.Debugf("unable to read %s", absPath)
return nil
3 years ago
}
for _, content := range contents {
if content.IsDir() {
dirs = append(dirs, path.Join(absPath, content.Name()))
} else {
if content.Name() == "index.html" {
indexPath := path.Join(absPath, content.Name())
file, err := os.ReadFile(indexPath)
3 years ago
if err != nil {
logrus.Debugf("unable to read %s, skipping", content.Name())
continue
3 years ago
}
3 years ago
if strings.Contains(string(file), generatedInDistribusi) {
logrus.Debugf("%s was not generated by distribusi, skipping", indexPath)
continue
3 years ago
}
}
3 years ago
files = append(files, path.Join(absPath, content.Name()))
}
}
if len(dirs) == 0 && len(files) == 0 {
return nil
}
if root != fpath {
href := "<a href='../'>../</a>"
div, err := mkDiv(c, "os/directory", href, "menu", false)
if err != nil {
return err
}
html = append(html, div)
}
for _, dir := range dirs {
fname := filepath.Base(dir)
skip, err := shouldSkip(c, fname, ignore)
if err != nil {
return err
}
if skip {
3 years ago
continue
}
mtype := "os/directory"
href := fmt.Sprintf("<a href='%s'>%s/</a>", fname, fname)
div, err := mkDiv(c, mtype, href, fname, false)
if err != nil {
return err
}
html = append(html, div)
progress.Add(1)
3 years ago
}
for _, file := range files {
fname := filepath.Base(file)
skip, err := shouldSkip(c, fname, ignore)
if err != nil {
return err
}
if skip {
3 years ago
continue
}
mtype, err := getMtype(file)
if err != nil {
logrus.Debugf("failed to read mimetype of %s", file)
continue
3 years ago
}
unknown, href, err := mkHref(c, file, mtype)
3 years ago
if err != nil {
logrus.Debugf("failed to generate href for %s", file)
continue
3 years ago
}
div, err := mkDiv(c, mtype, href, fname, unknown)
if err != nil {
logrus.Debugf("failed to generate div for %s", file)
continue
3 years ago
}
html = append(html, div)
progress.Add(1)
3 years ago
}
if err := writeIndex(absPath, html, c.String("css")); err != nil {
logrus.Debugf("unable to generated %s, skipping", path.Join(absPath, "index.html"))
return nil
3 years ago
}
allDirs = append(allDirs, strings.Join(dirs, " "))
}
return nil
}); err != nil {
return err
}
logrus.Debugf("generated files in following paths: %s", strings.Join(allDirs, " "))
return nil
}
// removeIndex safely removes an index.html, making sure to check that
// distribusi generated it.
func removeIndex(fpath string) error {
file, err := os.ReadFile(fpath)
if err != nil {
return err
}
if strings.Contains(string(file), generatedInDistribusi) {
if err := os.Remove(fpath); err != nil {
return err
}
return nil
}
return nil
}
// getMtype determines file mimetype which helps to arrange HTML tags
// appropriate for the file type.
func getMtype(fpath string) (string, error) {
mtype, err := mimetype.DetectFile(fpath)
if err != nil {
return "", err
}
return mtype.String(), nil
}
// genThumb generates an in-memory image thumbnail and encodes it into a base64
// representation which is suitable for embedding in HTML pages.
func genThumb(c *cli.Context, fpath, caption string) (string, error) {
imgSrc, err := imaging.Open(fpath, imaging.AutoOrientation(true))
if err != nil {
return "", err
}
img := imaging.Thumbnail(imgSrc, 450, 450, imaging.Lanczos)
buf := new(bytes.Buffer)
png.Encode(buf, img)
imgBase64Str := base64.StdEncoding.EncodeToString(buf.Bytes())
logrus.Debugf("successfully generated thumbnail for %s", fpath)
3 years ago
return imgBase64Str, nil
}
// getCaption retrieves an embedded image caption via exif-tool. If not
// exiftool is installed, we gracefully bail out. The caller is responsible for
// handling the alternative.
func getCaption(c *cli.Context, fpath string) (string, error) {
var caption string
if !exiftoolInstalled {
return "", nil
}
exif, err := exiftool.NewExiftool()
if err != nil {
return caption, fmt.Errorf("failed to initialise exiftool, saw %v", err)
}
defer exif.Close()
for _, finfo := range exif.ExtractMetadata(fpath) {
if finfo.Err != nil {
continue
}
for k, v := range finfo.Fields {
if k == "Comment" {
caption = fmt.Sprintf("%v", v)
}
}
}
if caption != "" {
logrus.Debugf("retrieved caption %s from %s", caption, fpath)
} else {
logrus.Debugf("no comment retrieved for %s", fpath)
}
3 years ago
return caption, nil
}
// parseMtype parses a mimetype string to simplify programmatic type lookups.
func parseMtype(mtype string) (string, string) {
stripCharset := strings.Split(mtype, ";")
splitTypes := strings.Split(stripCharset[0], "/")
ftype, stype := splitTypes[0], splitTypes[1]
return ftype, stype
}
// sliceContains checks if an element is present in a list.
func sliceContains(items []string, target string) bool {
for _, item := range items {
if item == target {
return true
}
}
return false
}
// trimFinalNewline trims newlines from the end of bytes just read from files.
func trimFinalNewline(contents []byte) string {
return strings.TrimSuffix(string(contents), "\n")
}
// trimAllNewlines removes all new lines.
func trimAllNewlines(contents string) string {
return strings.ReplaceAll(string(contents), "\n", "")
}
// mkHref figures out which href tag corresponds to which file by navigating
3 years ago
// the mimetype. If a type of file is unknown, this is signalled via the bool
// return value.
func mkHref(c *cli.Context, fpath string, mtype string) (bool, string, error) {
3 years ago
var href string
var hrefTemplate string
var strippedDebugOutput string
3 years ago
var unknown bool
fname := filepath.Base(fpath)
ftype, stype := parseMtype(mtype)
if ftype == "text" {
fcontents, err := os.ReadFile(fpath)
if err != nil {
return unknown, href, err
}
3 years ago
if stype == "html" {
hrefTemplate = htmlTags[ftype][stype]
href = fmt.Sprintf(hrefTemplate, fname, trimFinalNewline(fcontents))
strippedDebugOutput = fmt.Sprintf(hrefTemplate, fname, logStripMsg)
3 years ago
} else {
hrefTemplate = htmlTags[ftype]["generic"]
href = fmt.Sprintf(hrefTemplate, trimFinalNewline(fcontents))
strippedDebugOutput = fmt.Sprintf(hrefTemplate, logStripMsg)
3 years ago
}
} else if ftype == "image" {
caption, err := getCaption(c, fpath)
3 years ago
if err != nil {
return unknown, href, nil
}
if stype == "gif" {
hrefTemplate = htmlTags[ftype]["generic"]
href = fmt.Sprintf(hrefTemplate, fname, stype, fname, caption)
3 years ago
} else {
thumb, err := genThumb(c, fpath, caption)
if err != nil {
hrefTemplate = htmlTags[ftype]["generic"]
href = fmt.Sprintf(hrefTemplate, fname, stype, fname, caption)
logrus.Debugf("failed to generate thumbnail for %s, showing original image", fpath)
3 years ago
} else {
hrefTemplate = htmlTags[ftype]["thumbnail"]
href = fmt.Sprintf(hrefTemplate, fname, thumb, caption)
strippedDebugOutput = fmt.Sprintf(hrefTemplate, fname, logStripMsg, caption)
3 years ago
}
}
} else if ftype == "application" {
if stype == "pdf" {
hrefTemplate = htmlTags[ftype][stype]
href = fmt.Sprintf(hrefTemplate, fname, fname)
3 years ago
} else {
unknown = true
hrefTemplate = htmlTags["unknown"]["generic"]
href = fmt.Sprintf(hrefTemplate, stype, fname, fname)
3 years ago
}
} else if ftype == "audio" {
hrefTemplate = htmlTags[ftype]["generic"]
href = fmt.Sprintf(hrefTemplate, fname, stype)
3 years ago
} else if ftype == "video" {
hrefTemplate = htmlTags[ftype]["generic"]
href = fmt.Sprintf(hrefTemplate, fname, stype)
3 years ago
} else {
unknown = true
hrefTemplate = htmlTags["unknown"]["generic"]
href = fmt.Sprintf(hrefTemplate, stype, fname, fname)
3 years ago
}
if strippedDebugOutput != "" {
logrus.Debugf("%s was wrapped in: %s", fname, strippedDebugOutput)
} else {
logrus.Debugf("%s was wrapped in: %s", fname, href)
}
3 years ago
return unknown, href, nil
}
// mkDiv cosntructs a HTML div for inclusion in the generated index.html.
func mkDiv(c *cli.Context, mtype string, href, fname string, unknown bool) (string, error) {
var div string
var divTemplate string
var strippedDebugOutput string
3 years ago
filename := fmt.Sprintf("<span class='filename'>%s</span>", fname)
ftype, stype := parseMtype(mtype)
3 years ago
if ftype == "text" {
divTemplate = "<div id=\"%s\" class='%s'>%s%s</div>"
div = fmt.Sprintf(divTemplate, fname, ftype, href, filename)
strippedDebugOutput = fmt.Sprintf(divTemplate, fname, ftype, logStripMsg, filename)
3 years ago
} else if ftype == "os" {
if stype == "directory" {
divTemplate = "<div id=\"%s\" class='x-%s'>%s</div>"
div = fmt.Sprintf(divTemplate, fname, stype, href)
} else {
// don't include filename since link already has it
divTemplate = "<div id=\"%s\" class='%s'>%s</div>"
div = fmt.Sprintf(divTemplate, fname, ftype, href)
}
3 years ago
} else {
if unknown {
// don't include filename since link already has it
divTemplate = "<div id=\"%s\" class='%s'>%s</div>"
div = fmt.Sprintf(divTemplate, fname, ftype, href)
3 years ago
} else {
divTemplate = "<div id=\"%s\" class='%s'>%s%s</div>"
div = fmt.Sprintf(divTemplate, fname, ftype, href, filename)
strippedDebugOutput = fmt.Sprintf(divTemplate, fname, ftype, logStripMsg, filename)
3 years ago
}
}
if strippedDebugOutput != "" {
logrus.Debugf("%s was wrapped in: %s", fname, strippedDebugOutput)
} else {
logrus.Debugf("%s was wrapped in: %s", fname, div)
}
3 years ago
return div, nil
}
// writeIndex writes a new index.html.
func writeIndex(fpath string, html []string, styles string) error {
body := fmt.Sprintf(htmlBody, generatedInDistribusi, "", strings.Join(html, "\n"))
if styles != "" {
absPath, err := filepath.Abs(styles)
if err != nil {
return err
}
if _, err := os.Stat(absPath); !os.IsNotExist(err) {
contents, err := os.ReadFile(absPath)
if err != nil {
return nil
3 years ago
}
logrus.Debugf("loading custom styles from %s", absPath)
3 years ago
body = fmt.Sprintf(htmlBody, generatedInDistribusi, contents, strings.Join(html, "\n"))
}
}
HTMLPath := path.Join(fpath, "index.html")
contents := []byte(body)
3 years ago
if _, err := os.Stat(HTMLPath); err != nil {
if os.IsNotExist(err) {
if err := ioutil.WriteFile(HTMLPath, contents, 0644); err != nil {
logrus.Debugf("unable to write %s, skipping", HTMLPath)
return nil
3 years ago
}
} else {
logrus.Debugf("unable to read %s, skipping", HTMLPath)
return nil
3 years ago
}
} else {
file, err := os.ReadFile(HTMLPath)
if err != nil {
logrus.Debugf("unable to read %s, skipping", HTMLPath)
return nil
3 years ago
}
if strings.Contains(string(file), generatedInDistribusi) {
if err := ioutil.WriteFile(HTMLPath, contents, 0644); err != nil {
logrus.Debugf("unable to write %s, skipping", HTMLPath)
return nil
3 years ago
}
}
}
return nil
}
// serveHTTP serves a web server for browsing distribusi.
func serveHTTP(fpath string) error {
fs := http.FileServer(http.Dir(fpath))
http.Handle("/", fs)
logrus.Infof("distribusi serving from http://localhost%s", port)
3 years ago
if err := http.ListenAndServe(port, nil); err != nil {
3 years ago
return err
}
return nil
}
// wipeGeneratedFiles removes all distribusi generated files.
func wipeGeneratedFiles(dir string) error {
if err := filepath.WalkDir(dir, func(fpath string, dirEntry fs.DirEntry, err error) error {
fname := filepath.Base(fpath)
if fname == "index.html" {
if err := removeIndex(fpath); err != nil {
return err
}
logrus.Debugf("wiping %s as requested", fpath)
3 years ago
}
return nil
}); err != nil {
return err
}
return nil
}
// shouldSkip checks if paths should be skipped over.
func shouldSkip(c *cli.Context, fpath string, ignore []string) (bool, error) {
for _, pattern := range ignore {
match, err := filepath.Match(pattern, filepath.Base(fpath))
if err != nil {
return false, err
}
if match {
logrus.Debugf("skipping %s, matched %s", filepath.Base(fpath), pattern)
return true, nil
}
3 years ago
}
fpaths := strings.Split(fpath, "/")
for _, part := range fpaths {
if strings.HasPrefix(part, ".") {
if !c.Bool("hidden") {
logrus.Debugf("skipping %s, hidden directory", fpath)
return true, nil
3 years ago
}
}
}
return false, nil
3 years ago
}
// getLogFile creates a new log file for debug output.
func getLogFile() (*os.File, error) {
cTime := time.Now()
timeNow := fmt.Sprintf("%v-%v-%v", cTime.Hour(), cTime.Minute(), cTime.Second())
prefix := fmt.Sprintf("distribusi-go-%s", timeNow)
file, err := ioutil.TempFile("/tmp", prefix)
if err != nil {
return nil, err
}
logrus.Debugf("creating %s as debug log file", file.Name())
return file, nil
}
// mkProgressBar creates a progress bar.
func mkProgressBar() *progressbar.ProgressBar {
bar := progressbar.NewOptions(-1,
progressbar.OptionSetDescription("distribusifying..."),
progressbar.OptionSetWriter(ansi.NewAnsiStdout()),
progressbar.OptionEnableColorCodes(true),
progressbar.OptionShowCount(),
progressbar.OptionSpinnerType(9),
)
return bar
}