Varia's website https://varia.zone
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

626 lines
23 KiB

import copy
import datetime
import locale
import logging
import os
import re
from html import unescape
from urllib.parse import unquote, urljoin, urlparse, urlunparse
import pytz
from pelican.plugins import signals
from pelican.settings import DEFAULT_CONFIG
from pelican.utils import (deprecated_attribute, memoized, path_to_url,
posixize_path, sanitised_join, set_date_tzinfo,
slugify, truncate_html_words)
# Import these so that they're avalaible when you import from pelican.contents.
from pelican.urlwrappers import (Author, Category, Tag, URLWrapper) # NOQA
logger = logging.getLogger(__name__)
class Content:
"""Represents a content.
:param content: the string to parse, containing the original content.
:param metadata: the metadata associated to this page (optional).
:param settings: the settings dictionary (optional).
:param source_path: The location of the source of this content (if any).
:param context: The shared context between generators.
"""
@deprecated_attribute(old='filename', new='source_path', since=(3, 2, 0))
def filename():
return None
def __init__(self, content, metadata=None, settings=None,
source_path=None, context=None):
if metadata is None:
metadata = {}
if settings is None:
settings = copy.deepcopy(DEFAULT_CONFIG)
self.settings = settings
self._content = content
if context is None:
context = {}
self._context = context
self.translations = []
local_metadata = dict()
local_metadata.update(metadata)
# set metadata as attributes
for key, value in local_metadata.items():
if key in ('save_as', 'url'):
key = 'override_' + key
setattr(self, key.lower(), value)
# also keep track of the metadata attributes available
self.metadata = local_metadata
# default template if it's not defined in page
self.template = self._get_template()
# First, read the authors from "authors", if not, fallback to "author"
# and if not use the settings defined one, if any.
if not hasattr(self, 'author'):
if hasattr(self, 'authors'):
self.author = self.authors[0]
elif 'AUTHOR' in settings:
self.author = Author(settings['AUTHOR'], settings)
if not hasattr(self, 'authors') and hasattr(self, 'author'):
self.authors = [self.author]
# XXX Split all the following code into pieces, there is too much here.
# manage languages
self.in_default_lang = True
if 'DEFAULT_LANG' in settings:
default_lang = settings['DEFAULT_LANG'].lower()
if not hasattr(self, 'lang'):
self.lang = default_lang
self.in_default_lang = (self.lang == default_lang)
# create the slug if not existing, generate slug according to
# setting of SLUG_ATTRIBUTE
if not hasattr(self, 'slug'):
if (settings['SLUGIFY_SOURCE'] == 'title' and
hasattr(self, 'title')):
value = self.title
elif (settings['SLUGIFY_SOURCE'] == 'basename' and
source_path is not None):
value = os.path.basename(os.path.splitext(source_path)[0])
else:
value = None
if value is not None:
self.slug = slugify(
value,
regex_subs=settings.get('SLUG_REGEX_SUBSTITUTIONS', []),
preserve_case=settings.get('SLUGIFY_PRESERVE_CASE', False),
use_unicode=settings.get('SLUGIFY_USE_UNICODE', False))
self.source_path = source_path
self.relative_source_path = self.get_relative_source_path()
# manage the date format
if not hasattr(self, 'date_format'):
if hasattr(self, 'lang') and self.lang in settings['DATE_FORMATS']:
self.date_format = settings['DATE_FORMATS'][self.lang]
else:
self.date_format = settings['DEFAULT_DATE_FORMAT']
if isinstance(self.date_format, tuple):
locale_string = self.date_format[0]
locale.setlocale(locale.LC_ALL, locale_string)
self.date_format = self.date_format[1]
# manage timezone
default_timezone = settings.get('TIMEZONE', 'UTC')
timezone = getattr(self, 'timezone', default_timezone)
self.timezone = pytz.timezone(timezone)
if hasattr(self, 'date'):
self.date = set_date_tzinfo(self.date, timezone)
self.locale_date = self.date.strftime(self.date_format)
if hasattr(self, 'modified'):
self.modified = set_date_tzinfo(self.modified, timezone)
self.locale_modified = self.modified.strftime(self.date_format)
# manage status
if not hasattr(self, 'status'):
# Previous default of None broke comment plugins and perhaps others
self.status = getattr(self, 'default_status', '')
# store the summary metadata if it is set
if 'summary' in metadata:
self._summary = metadata['summary']
signals.content_object_init.send(self)
def __str__(self):
return self.source_path or repr(self)
def _has_valid_mandatory_properties(self):
"""Test mandatory properties are set."""
for prop in self.mandatory_properties:
if not hasattr(self, prop):
logger.error(
"Skipping %s: could not find information about '%s'",
self, prop)
return False
return True
def _has_valid_save_as(self):
"""Return true if save_as doesn't write outside output path, false
otherwise."""
try:
output_path = self.settings["OUTPUT_PATH"]
except KeyError:
# we cannot check
return True
try:
sanitised_join(output_path, self.save_as)
except RuntimeError: # outside output_dir
logger.error(
"Skipping %s: file %r would be written outside output path",
self,
self.save_as,
)
return False
return True
def _has_valid_status(self):
if hasattr(self, 'allowed_statuses'):
if self.status not in self.allowed_statuses:
logger.error(
"Unknown status '%s' for file %s, skipping it.",
self.status,
self
)
return False
# if undefined we allow all
return True
def is_valid(self):
"""Validate Content"""
# Use all() to not short circuit and get results of all validations
return all([self._has_valid_mandatory_properties(),
self._has_valid_save_as(),
self._has_valid_status()])
@property
def url_format(self):
"""Returns the URL, formatted with the proper values"""
metadata = copy.copy(self.metadata)
path = self.metadata.get('path', self.get_relative_source_path())
metadata.update({
'path': path_to_url(path),
'slug': getattr(self, 'slug', ''),
'lang': getattr(self, 'lang', 'en'),
'date': getattr(self, 'date', datetime.datetime.now()),
'author': self.author.slug if hasattr(self, 'author') else '',
'category': self.category.slug if hasattr(self, 'category') else ''
})
return metadata
def _expand_settings(self, key, klass=None):
if not klass:
klass = self.__class__.__name__
fq_key = ('{}_{}'.format(klass, key)).upper()
return self.settings[fq_key].format(**self.url_format)
def get_url_setting(self, key):
if hasattr(self, 'override_' + key):
return getattr(self, 'override_' + key)
key = key if self.in_default_lang else 'lang_%s' % key
return self._expand_settings(key)
def _link_replacer(self, siteurl, m):
what = m.group('what')
value = urlparse(m.group('value'))
path = value.path
origin = m.group('path')
# urllib.parse.urljoin() produces `a.html` for urljoin("..", "a.html")
# so if RELATIVE_URLS are enabled, we fall back to os.path.join() to
# properly get `../a.html`. However, os.path.join() produces
# `baz/http://foo/bar.html` for join("baz", "http://foo/bar.html")
# instead of correct "http://foo/bar.html", so one has to pick a side
# as there is no silver bullet.
if self.settings['RELATIVE_URLS']:
joiner = os.path.join
else:
joiner = urljoin
# However, it's not *that* simple: urljoin("blog", "index.html")
# produces just `index.html` instead of `blog/index.html` (unlike
# os.path.join()), so in order to get a correct answer one needs to
# append a trailing slash to siteurl in that case. This also makes
# the new behavior fully compatible with Pelican 3.7.1.
if not siteurl.endswith('/'):
siteurl += '/'
# XXX Put this in a different location.
if what in {'filename', 'static', 'attach'}:
def _get_linked_content(key, url):
nonlocal value
def _find_path(path):
if path.startswith('/'):
path = path[1:]
else:
# relative to the source path of this content
path = self.get_relative_source_path(
os.path.join(self.relative_dir, path)
)
return self._context[key].get(path, None)
# try path
result = _find_path(url.path)
if result is not None:
return result
# try unquoted path
result = _find_path(unquote(url.path))
if result is not None:
return result
# try html unescaped url
unescaped_url = urlparse(unescape(url.geturl()))
result = _find_path(unescaped_url.path)
if result is not None:
value = unescaped_url
return result
# check if a static file is linked with {filename}
if what == 'filename' and key == 'generated_content':
linked_content = _get_linked_content('static_content', value)
if linked_content:
logger.warning(
'{filename} used for linking to static'
' content %s in %s. Use {static} instead',
value.path,
self.get_relative_source_path())
return linked_content
return None
if what == 'filename':
key = 'generated_content'
else:
key = 'static_content'
linked_content = _get_linked_content(key, value)
if linked_content:
if what == 'attach':
linked_content.attach_to(self)
origin = joiner(siteurl, linked_content.url)
origin = origin.replace('\\', '/') # for Windows paths.
else:
logger.warning(
"Unable to find '%s', skipping url replacement.",
value.geturl(), extra={
'limit_msg': ("Other resources were not found "
"and their urls not replaced")})
elif what == 'category':
origin = joiner(siteurl, Category(path, self.settings).url)
elif what == 'tag':
origin = joiner(siteurl, Tag(path, self.settings).url)
elif what == 'index':
origin = joiner(siteurl, self.settings['INDEX_SAVE_AS'])
elif what == 'author':
origin = joiner(siteurl, Author(path, self.settings).url)
else:
logger.warning(
"Replacement Indicator '%s' not recognized, "
"skipping replacement",
what)
# keep all other parts, such as query, fragment, etc.
parts = list(value)
parts[2] = origin
origin = urlunparse(parts)
return ''.join((m.group('markup'), m.group('quote'), origin,
m.group('quote')))
def _get_intrasite_link_regex(self):
intrasite_link_regex = self.settings['INTRASITE_LINK_REGEX']
regex = r"""
(?P<markup><[^\>]+ # match tag with all url-value attributes
(?:href|src|poster|data|cite|formaction|action)\s*=\s*)
(?P<quote>["\']) # require value to be quoted
(?P<path>{}(?P<value>.*?)) # the url value
\2""".format(intrasite_link_regex)
return re.compile(regex, re.X)
def _update_content(self, content, siteurl):
"""Update the content attribute.
Change all the relative paths of the content to relative paths
suitable for the output content.
:param content: content resource that will be passed to the templates.
:param siteurl: siteurl which is locally generated by the writer in
case of RELATIVE_URLS.
"""
if not content:
return content
hrefs = self._get_intrasite_link_regex()
return hrefs.sub(lambda m: self._link_replacer(siteurl, m), content)
def get_static_links(self):
static_links = set()
hrefs = self._get_intrasite_link_regex()
for m in hrefs.finditer(self._content):
what = m.group('what')
value = urlparse(m.group('value'))
path = value.path
if what not in {'static', 'attach'}:
continue
if path.startswith('/'):
path = path[1:]
else:
# relative to the source path of this content
path = self.get_relative_source_path(
os.path.join(self.relative_dir, path)
)
path = path.replace('%20', ' ')
static_links.add(path)
return static_links
def get_siteurl(self):
return self._context.get('localsiteurl', '')
@memoized
def get_content(self, siteurl):
if hasattr(self, '_get_content'):
content = self._get_content()
else:
content = self._content
return self._update_content(content, siteurl)
@property
def content(self):
return self.get_content(self.get_siteurl())
@memoized
def get_summary(self, siteurl):
"""Returns the summary of an article.
This is based on the summary metadata if set, otherwise truncate the
content.
"""
if 'summary' in self.metadata:
return self.metadata['summary']
if self.settings['SUMMARY_MAX_LENGTH'] is None:
return self.content
return truncate_html_words(self.content,
self.settings['SUMMARY_MAX_LENGTH'],
self.settings['SUMMARY_END_SUFFIX'])
@property
def summary(self):
return self.get_summary(self.get_siteurl())
def _get_summary(self):
"""deprecated function to access summary"""
logger.warning('_get_summary() has been deprecated since 3.6.4. '
'Use the summary decorator instead')
return self.summary
@summary.setter
def summary(self, value):
"""Dummy function"""
pass
@property
def status(self):
return self._status
@status.setter
def status(self, value):
# TODO maybe typecheck
self._status = value.lower()
@property
def url(self):
return self.get_url_setting('url')
@property
def save_as(self):
return self.get_url_setting('save_as')
def _get_template(self):
if hasattr(self, 'template') and self.template is not None:
return self.template
else:
return self.default_template
def get_relative_source_path(self, source_path=None):
"""Return the relative path (from the content path) to the given
source_path.
If no source path is specified, use the source path of this
content object.
"""
if not source_path:
source_path = self.source_path
if source_path is None:
return None
return posixize_path(
os.path.relpath(
os.path.abspath(os.path.join(
self.settings['PATH'],
source_path)),
os.path.abspath(self.settings['PATH'])
))
@property
def relative_dir(self):
return posixize_path(
os.path.dirname(
os.path.relpath(
os.path.abspath(self.source_path),
os.path.abspath(self.settings['PATH']))))
def refresh_metadata_intersite_links(self):
for key in self.settings['FORMATTED_FIELDS']:
if key in self.metadata and key != 'summary':
value = self._update_content(
self.metadata[key],
self.get_siteurl()
)
self.metadata[key] = value
setattr(self, key.lower(), value)
# _summary is an internal variable that some plugins may be writing to,
# so ensure changes to it are picked up
if ('summary' in self.settings['FORMATTED_FIELDS'] and
'summary' in self.metadata):
self._summary = self._update_content(
self._summary,
self.get_siteurl()
)
self.metadata['summary'] = self._summary
class Page(Content):
mandatory_properties = ('title',)
allowed_statuses = ('published', 'hidden', 'draft')
default_status = 'published'
default_template = 'page'
def _expand_settings(self, key):
klass = 'draft_page' if self.status == 'draft' else None
return super()._expand_settings(key, klass)
class Article(Content):
mandatory_properties = ('title', 'date', 'category')
allowed_statuses = ('published', 'draft')
default_status = 'published'
default_template = 'article'
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# handle WITH_FUTURE_DATES (designate article to draft based on date)
if not self.settings['WITH_FUTURE_DATES'] and hasattr(self, 'date'):
if self.date.tzinfo is None:
now = datetime.datetime.now()
else:
now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
if self.date > now:
self.status = 'draft'
# if we are a draft and there is no date provided, set max datetime
if not hasattr(self, 'date') and self.status == 'draft':
self.date = datetime.datetime.max.replace(tzinfo=self.timezone)
def _expand_settings(self, key):
klass = 'draft' if self.status == 'draft' else 'article'
return super()._expand_settings(key, klass)
class Static(Content):
mandatory_properties = ('title',)
default_status = 'published'
default_template = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._output_location_referenced = False
@deprecated_attribute(old='filepath', new='source_path', since=(3, 2, 0))
def filepath():
return None
@deprecated_attribute(old='src', new='source_path', since=(3, 2, 0))
def src():
return None
@deprecated_attribute(old='dst', new='save_as', since=(3, 2, 0))
def dst():
return None
@property
def url(self):
# Note when url has been referenced, so we can avoid overriding it.
self._output_location_referenced = True
return super().url
@property
def save_as(self):
# Note when save_as has been referenced, so we can avoid overriding it.
self._output_location_referenced = True
return super().save_as
def attach_to(self, content):
"""Override our output directory with that of the given content object.
"""
# Determine our file's new output path relative to the linking
# document. If it currently lives beneath the linking
# document's source directory, preserve that relationship on output.
# Otherwise, make it a sibling.
linking_source_dir = os.path.dirname(content.source_path)
tail_path = os.path.relpath(self.source_path, linking_source_dir)
if tail_path.startswith(os.pardir + os.sep):
tail_path = os.path.basename(tail_path)
new_save_as = os.path.join(
os.path.dirname(content.save_as), tail_path)
# We do not build our new url by joining tail_path with the linking
# document's url, because we cannot know just by looking at the latter
# whether it points to the document itself or to its parent directory.
# (An url like 'some/content' might mean a directory named 'some'
# with a file named 'content', or it might mean a directory named
# 'some/content' with a file named 'index.html'.) Rather than trying
# to figure it out by comparing the linking document's url and save_as
# path, we simply build our new url from our new save_as path.
new_url = path_to_url(new_save_as)
def _log_reason(reason):
logger.warning(
"The {attach} link in %s cannot relocate "
"%s because %s. Falling back to "
"{filename} link behavior instead.",
content.get_relative_source_path(),
self.get_relative_source_path(), reason,
extra={'limit_msg': "More {attach} warnings silenced."})
# We never override an override, because we don't want to interfere
# with user-defined overrides that might be in EXTRA_PATH_METADATA.
if hasattr(self, 'override_save_as') or hasattr(self, 'override_url'):
if new_save_as != self.save_as or new_url != self.url:
_log_reason("its output location was already overridden")
return
# We never change an output path that has already been referenced,
# because we don't want to break links that depend on that path.
if self._output_location_referenced:
if new_save_as != self.save_as or new_url != self.url:
_log_reason("another link already referenced its location")
return
self.override_save_as = new_save_as
self.override_url = new_url