forked from varia/varia.website
451 lines
17 KiB
Python
451 lines
17 KiB
Python
|
"""i18n_subsites plugin creates i18n-ized subsites of the default site
|
||
|
|
|
||
|
|
This plugin is designed for Pelican 3.4 and later
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
import os
|
||
|
|
import six
|
||
|
|
import logging
|
||
|
|
import posixpath
|
||
|
|
|
||
|
|
from copy import copy
|
||
|
|
from itertools import chain
|
||
|
|
from operator import attrgetter
|
||
|
|
from collections import OrderedDict
|
||
|
|
from contextlib import contextmanager
|
||
|
|
from six.moves.urllib.parse import urlparse
|
||
|
|
|
||
|
|
import gettext
|
||
|
|
import locale
|
||
|
|
|
||
|
|
from pelican import signals
|
||
|
|
from pelican.generators import ArticlesGenerator, PagesGenerator
|
||
|
|
from pelican.settings import configure_settings
|
||
|
|
from pelican.contents import Draft
|
||
|
|
|
||
|
|
|
||
|
|
# Global vars
|
||
|
|
_MAIN_SETTINGS = None # settings dict of the main Pelican instance
|
||
|
|
_MAIN_LANG = None # lang of the main Pelican instance
|
||
|
|
_MAIN_SITEURL = None # siteurl of the main Pelican instance
|
||
|
|
_MAIN_STATIC_FILES = None # list of Static instances the main Pelican instance
|
||
|
|
_SUBSITE_QUEUE = {} # map: lang -> settings overrides
|
||
|
|
_SITE_DB = OrderedDict() # OrderedDict: lang -> siteurl
|
||
|
|
_SITES_RELPATH_DB = {} # map: (lang, base_lang) -> relpath
|
||
|
|
# map: generator -> list of removed contents that need interlinking
|
||
|
|
_GENERATOR_DB = {}
|
||
|
|
_NATIVE_CONTENT_URL_DB = {} # map: source_path -> content in its native lang
|
||
|
|
_LOGGER = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
@contextmanager
|
||
|
|
def temporary_locale(temp_locale=None):
|
||
|
|
'''Enable code to run in a context with a temporary locale
|
||
|
|
|
||
|
|
Resets the locale back when exiting context.
|
||
|
|
Can set a temporary locale if provided
|
||
|
|
'''
|
||
|
|
orig_locale = locale.setlocale(locale.LC_ALL)
|
||
|
|
if temp_locale is not None:
|
||
|
|
locale.setlocale(locale.LC_ALL, temp_locale)
|
||
|
|
yield
|
||
|
|
locale.setlocale(locale.LC_ALL, orig_locale)
|
||
|
|
|
||
|
|
|
||
|
|
def initialize_dbs(settings):
|
||
|
|
'''Initialize internal DBs using the Pelican settings dict
|
||
|
|
|
||
|
|
This clears the DBs for e.g. autoreload mode to work
|
||
|
|
'''
|
||
|
|
global _MAIN_SETTINGS, _MAIN_SITEURL, _MAIN_LANG, _SUBSITE_QUEUE
|
||
|
|
_MAIN_SETTINGS = settings
|
||
|
|
_MAIN_LANG = settings['DEFAULT_LANG']
|
||
|
|
_MAIN_SITEURL = settings['SITEURL']
|
||
|
|
_SUBSITE_QUEUE = settings.get('I18N_SUBSITES', {}).copy()
|
||
|
|
prepare_site_db_and_overrides()
|
||
|
|
# clear databases in case of autoreload mode
|
||
|
|
_SITES_RELPATH_DB.clear()
|
||
|
|
_NATIVE_CONTENT_URL_DB.clear()
|
||
|
|
_GENERATOR_DB.clear()
|
||
|
|
|
||
|
|
|
||
|
|
def prepare_site_db_and_overrides():
|
||
|
|
'''Prepare overrides and create _SITE_DB
|
||
|
|
|
||
|
|
_SITE_DB.keys() need to be ready for filter_translations
|
||
|
|
'''
|
||
|
|
_SITE_DB.clear()
|
||
|
|
_SITE_DB[_MAIN_LANG] = _MAIN_SITEURL
|
||
|
|
# make sure it works for both root-relative and absolute
|
||
|
|
main_siteurl = '/' if _MAIN_SITEURL == '' else _MAIN_SITEURL
|
||
|
|
for lang, overrides in _SUBSITE_QUEUE.items():
|
||
|
|
if 'SITEURL' not in overrides:
|
||
|
|
overrides['SITEURL'] = posixpath.join(main_siteurl, lang)
|
||
|
|
_SITE_DB[lang] = overrides['SITEURL']
|
||
|
|
# default subsite hierarchy
|
||
|
|
if 'OUTPUT_PATH' not in overrides:
|
||
|
|
overrides['OUTPUT_PATH'] = os.path.join(
|
||
|
|
_MAIN_SETTINGS['OUTPUT_PATH'], lang)
|
||
|
|
if 'CACHE_PATH' not in overrides:
|
||
|
|
overrides['CACHE_PATH'] = os.path.join(
|
||
|
|
_MAIN_SETTINGS['CACHE_PATH'], lang)
|
||
|
|
if 'STATIC_PATHS' not in overrides:
|
||
|
|
overrides['STATIC_PATHS'] = []
|
||
|
|
if ('THEME' not in overrides and 'THEME_STATIC_DIR' not in overrides and
|
||
|
|
'THEME_STATIC_PATHS' not in overrides):
|
||
|
|
relpath = relpath_to_site(lang, _MAIN_LANG)
|
||
|
|
overrides['THEME_STATIC_DIR'] = posixpath.join(
|
||
|
|
relpath, _MAIN_SETTINGS['THEME_STATIC_DIR'])
|
||
|
|
overrides['THEME_STATIC_PATHS'] = []
|
||
|
|
# to change what is perceived as translations
|
||
|
|
overrides['DEFAULT_LANG'] = lang
|
||
|
|
|
||
|
|
|
||
|
|
def subscribe_filter_to_signals(settings):
|
||
|
|
'''Subscribe content filter to requested signals'''
|
||
|
|
for sig in settings.get('I18N_FILTER_SIGNALS', []):
|
||
|
|
sig.connect(filter_contents_translations)
|
||
|
|
|
||
|
|
|
||
|
|
def initialize_plugin(pelican_obj):
|
||
|
|
'''Initialize plugin variables and Pelican settings'''
|
||
|
|
if _MAIN_SETTINGS is None:
|
||
|
|
initialize_dbs(pelican_obj.settings)
|
||
|
|
subscribe_filter_to_signals(pelican_obj.settings)
|
||
|
|
|
||
|
|
|
||
|
|
def get_site_path(url):
|
||
|
|
'''Get the path component of an url, excludes siteurl
|
||
|
|
|
||
|
|
also normalizes '' to '/' for relpath to work,
|
||
|
|
otherwise it could be interpreted as a relative filesystem path
|
||
|
|
'''
|
||
|
|
path = urlparse(url).path
|
||
|
|
if path == '':
|
||
|
|
path = '/'
|
||
|
|
return path
|
||
|
|
|
||
|
|
|
||
|
|
def relpath_to_site(lang, target_lang):
|
||
|
|
'''Get relative path from siteurl of lang to siteurl of base_lang
|
||
|
|
|
||
|
|
the output is cached in _SITES_RELPATH_DB
|
||
|
|
'''
|
||
|
|
path = _SITES_RELPATH_DB.get((lang, target_lang), None)
|
||
|
|
if path is None:
|
||
|
|
siteurl = _SITE_DB.get(lang, _MAIN_SITEURL)
|
||
|
|
target_siteurl = _SITE_DB.get(target_lang, _MAIN_SITEURL)
|
||
|
|
path = posixpath.relpath(get_site_path(target_siteurl),
|
||
|
|
get_site_path(siteurl))
|
||
|
|
_SITES_RELPATH_DB[(lang, target_lang)] = path
|
||
|
|
return path
|
||
|
|
|
||
|
|
|
||
|
|
def save_generator(generator):
|
||
|
|
'''Save the generator for later use
|
||
|
|
|
||
|
|
initialize the removed content list
|
||
|
|
'''
|
||
|
|
_GENERATOR_DB[generator] = []
|
||
|
|
|
||
|
|
|
||
|
|
def article2draft(article):
|
||
|
|
'''Transform an Article to Draft'''
|
||
|
|
draft = Draft(article._content, article.metadata, article.settings,
|
||
|
|
article.source_path, article._context)
|
||
|
|
draft.status = 'draft'
|
||
|
|
return draft
|
||
|
|
|
||
|
|
|
||
|
|
def page2hidden_page(page):
|
||
|
|
'''Transform a Page to a hidden Page'''
|
||
|
|
page.status = 'hidden'
|
||
|
|
return page
|
||
|
|
|
||
|
|
|
||
|
|
class GeneratorInspector(object):
|
||
|
|
'''Inspector of generator instances'''
|
||
|
|
|
||
|
|
generators_info = {
|
||
|
|
ArticlesGenerator: {
|
||
|
|
'translations_lists': ['translations', 'drafts_translations'],
|
||
|
|
'contents_lists': [('articles', 'drafts')],
|
||
|
|
'hiding_func': article2draft,
|
||
|
|
'policy': 'I18N_UNTRANSLATED_ARTICLES',
|
||
|
|
},
|
||
|
|
PagesGenerator: {
|
||
|
|
'translations_lists': ['translations', 'hidden_translations'],
|
||
|
|
'contents_lists': [('pages', 'hidden_pages')],
|
||
|
|
'hiding_func': page2hidden_page,
|
||
|
|
'policy': 'I18N_UNTRANSLATED_PAGES',
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
def __init__(self, generator):
|
||
|
|
'''Identify the best known class of the generator instance
|
||
|
|
|
||
|
|
The class '''
|
||
|
|
self.generator = generator
|
||
|
|
self.generators_info.update(generator.settings.get(
|
||
|
|
'I18N_GENERATORS_INFO', {}))
|
||
|
|
for cls in generator.__class__.__mro__:
|
||
|
|
if cls in self.generators_info:
|
||
|
|
self.info = self.generators_info[cls]
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
self.info = {}
|
||
|
|
|
||
|
|
def translations_lists(self):
|
||
|
|
'''Iterator over lists of content translations'''
|
||
|
|
return (getattr(self.generator, name) for name in
|
||
|
|
self.info.get('translations_lists', []))
|
||
|
|
|
||
|
|
def contents_list_pairs(self):
|
||
|
|
'''Iterator over pairs of normal and hidden contents'''
|
||
|
|
return (tuple(getattr(self.generator, name) for name in names)
|
||
|
|
for names in self.info.get('contents_lists', []))
|
||
|
|
|
||
|
|
def hiding_function(self):
|
||
|
|
'''Function for transforming content to a hidden version'''
|
||
|
|
hiding_func = self.info.get('hiding_func', lambda x: x)
|
||
|
|
return hiding_func
|
||
|
|
|
||
|
|
def untranslated_policy(self, default):
|
||
|
|
'''Get the policy for untranslated content'''
|
||
|
|
return self.generator.settings.get(self.info.get('policy', None),
|
||
|
|
default)
|
||
|
|
|
||
|
|
def all_contents(self):
|
||
|
|
'''Iterator over all contents'''
|
||
|
|
translations_iterator = chain(*self.translations_lists())
|
||
|
|
return chain(translations_iterator,
|
||
|
|
*(pair[i] for pair in self.contents_list_pairs()
|
||
|
|
for i in (0, 1)))
|
||
|
|
|
||
|
|
|
||
|
|
def filter_contents_translations(generator):
|
||
|
|
'''Filter the content and translations lists of a generator
|
||
|
|
|
||
|
|
Filters out
|
||
|
|
1) translations which will be generated in a different site
|
||
|
|
2) content that is not in the language of the currently
|
||
|
|
generated site but in that of a different site, content in a
|
||
|
|
language which has no site is generated always. The filtering
|
||
|
|
method bay be modified by the respective untranslated policy
|
||
|
|
'''
|
||
|
|
inspector = GeneratorInspector(generator)
|
||
|
|
current_lang = generator.settings['DEFAULT_LANG']
|
||
|
|
langs_with_sites = _SITE_DB.keys()
|
||
|
|
removed_contents = _GENERATOR_DB[generator]
|
||
|
|
|
||
|
|
for translations in inspector.translations_lists():
|
||
|
|
for translation in translations[:]: # copy to be able to remove
|
||
|
|
if translation.lang in langs_with_sites:
|
||
|
|
translations.remove(translation)
|
||
|
|
removed_contents.append(translation)
|
||
|
|
|
||
|
|
hiding_func = inspector.hiding_function()
|
||
|
|
untrans_policy = inspector.untranslated_policy(default='hide')
|
||
|
|
for (contents, other_contents) in inspector.contents_list_pairs():
|
||
|
|
for content in other_contents: # save any hidden native content first
|
||
|
|
if content.lang == current_lang: # in native lang
|
||
|
|
# save the native URL attr formatted in the current locale
|
||
|
|
_NATIVE_CONTENT_URL_DB[content.source_path] = content.url
|
||
|
|
for content in contents[:]: # copy for removing in loop
|
||
|
|
if content.lang == current_lang: # in native lang
|
||
|
|
# save the native URL attr formatted in the current locale
|
||
|
|
_NATIVE_CONTENT_URL_DB[content.source_path] = content.url
|
||
|
|
elif content.lang in langs_with_sites and untrans_policy != 'keep':
|
||
|
|
contents.remove(content)
|
||
|
|
if untrans_policy == 'hide':
|
||
|
|
other_contents.append(hiding_func(content))
|
||
|
|
elif untrans_policy == 'remove':
|
||
|
|
removed_contents.append(content)
|
||
|
|
|
||
|
|
|
||
|
|
def install_templates_translations(generator):
|
||
|
|
'''Install gettext translations in the jinja2.Environment
|
||
|
|
|
||
|
|
Only if the 'jinja2.ext.i18n' jinja2 extension is enabled
|
||
|
|
the translations for the current DEFAULT_LANG are installed.
|
||
|
|
'''
|
||
|
|
if 'JINJA_ENVIRONMENT' in generator.settings: # pelican 3.7+
|
||
|
|
jinja_extensions = generator.settings['JINJA_ENVIRONMENT'].get(
|
||
|
|
'extensions', [])
|
||
|
|
else:
|
||
|
|
jinja_extensions = generator.settings['JINJA_EXTENSIONS']
|
||
|
|
|
||
|
|
if 'jinja2.ext.i18n' in jinja_extensions:
|
||
|
|
domain = generator.settings.get('I18N_GETTEXT_DOMAIN', 'messages')
|
||
|
|
localedir = generator.settings.get('I18N_GETTEXT_LOCALEDIR')
|
||
|
|
if localedir is None:
|
||
|
|
localedir = os.path.join(generator.theme, 'translations')
|
||
|
|
current_lang = generator.settings['DEFAULT_LANG']
|
||
|
|
if current_lang == generator.settings.get('I18N_TEMPLATES_LANG',
|
||
|
|
_MAIN_LANG):
|
||
|
|
translations = gettext.NullTranslations()
|
||
|
|
else:
|
||
|
|
langs = [current_lang]
|
||
|
|
try:
|
||
|
|
translations = gettext.translation(domain, localedir, langs)
|
||
|
|
except (IOError, OSError):
|
||
|
|
_LOGGER.error((
|
||
|
|
"Cannot find translations for language '{}' in '{}' with "
|
||
|
|
"domain '{}'. Installing NullTranslations.").format(
|
||
|
|
langs[0], localedir, domain))
|
||
|
|
translations = gettext.NullTranslations()
|
||
|
|
newstyle = generator.settings.get('I18N_GETTEXT_NEWSTYLE', True)
|
||
|
|
generator.env.install_gettext_translations(translations, newstyle)
|
||
|
|
|
||
|
|
|
||
|
|
def add_variables_to_context(generator):
|
||
|
|
'''Adds useful iterable variables to template context'''
|
||
|
|
context = generator.context # minimize attr lookup
|
||
|
|
context['relpath_to_site'] = relpath_to_site
|
||
|
|
context['main_siteurl'] = _MAIN_SITEURL
|
||
|
|
context['main_lang'] = _MAIN_LANG
|
||
|
|
context['lang_siteurls'] = _SITE_DB
|
||
|
|
current_lang = generator.settings['DEFAULT_LANG']
|
||
|
|
extra_siteurls = _SITE_DB.copy()
|
||
|
|
extra_siteurls.pop(current_lang)
|
||
|
|
context['extra_siteurls'] = extra_siteurls
|
||
|
|
|
||
|
|
|
||
|
|
def interlink_translations(content):
|
||
|
|
'''Link content to translations in their main language
|
||
|
|
|
||
|
|
so the URL (including localized month names) of the different subsites
|
||
|
|
will be honored
|
||
|
|
'''
|
||
|
|
lang = content.lang
|
||
|
|
# sort translations by lang
|
||
|
|
content.translations.sort(key=attrgetter('lang'))
|
||
|
|
for translation in content.translations:
|
||
|
|
relpath = relpath_to_site(lang, translation.lang)
|
||
|
|
url = _NATIVE_CONTENT_URL_DB[translation.source_path]
|
||
|
|
translation.override_url = posixpath.join(relpath, url)
|
||
|
|
|
||
|
|
|
||
|
|
def interlink_translated_content(generator):
|
||
|
|
'''Make translations link to the native locations
|
||
|
|
|
||
|
|
for generators that may contain translated content
|
||
|
|
'''
|
||
|
|
inspector = GeneratorInspector(generator)
|
||
|
|
for content in inspector.all_contents():
|
||
|
|
interlink_translations(content)
|
||
|
|
|
||
|
|
|
||
|
|
def interlink_removed_content(generator):
|
||
|
|
'''For all contents removed from generation queue update interlinks
|
||
|
|
|
||
|
|
link to the native location
|
||
|
|
'''
|
||
|
|
current_lang = generator.settings['DEFAULT_LANG']
|
||
|
|
for content in _GENERATOR_DB[generator]:
|
||
|
|
url = _NATIVE_CONTENT_URL_DB[content.source_path]
|
||
|
|
relpath = relpath_to_site(current_lang, content.lang)
|
||
|
|
content.override_url = posixpath.join(relpath, url)
|
||
|
|
|
||
|
|
|
||
|
|
def interlink_static_files(generator):
|
||
|
|
'''Add links to static files in the main site if necessary'''
|
||
|
|
if generator.settings['STATIC_PATHS'] != []:
|
||
|
|
return # customized STATIC_PATHS
|
||
|
|
filenames = generator.context['filenames'] # minimize attr lookup
|
||
|
|
relpath = relpath_to_site(generator.settings['DEFAULT_LANG'], _MAIN_LANG)
|
||
|
|
for staticfile in _MAIN_STATIC_FILES:
|
||
|
|
if staticfile.get_relative_source_path() not in filenames:
|
||
|
|
staticfile = copy(staticfile) # prevent override in main site
|
||
|
|
staticfile.override_url = posixpath.join(relpath, staticfile.url)
|
||
|
|
generator.add_source_path(staticfile)
|
||
|
|
|
||
|
|
|
||
|
|
def save_main_static_files(static_generator):
|
||
|
|
'''Save the static files generated for the main site'''
|
||
|
|
global _MAIN_STATIC_FILES
|
||
|
|
# test just for current lang as settings change in autoreload mode
|
||
|
|
if static_generator.settings['DEFAULT_LANG'] == _MAIN_LANG:
|
||
|
|
_MAIN_STATIC_FILES = static_generator.staticfiles
|
||
|
|
|
||
|
|
|
||
|
|
def update_generators():
|
||
|
|
'''Update the context of all generators
|
||
|
|
|
||
|
|
Ads useful variables and translations into the template context
|
||
|
|
and interlink translations
|
||
|
|
'''
|
||
|
|
for generator in _GENERATOR_DB.keys():
|
||
|
|
install_templates_translations(generator)
|
||
|
|
add_variables_to_context(generator)
|
||
|
|
interlink_static_files(generator)
|
||
|
|
interlink_removed_content(generator)
|
||
|
|
interlink_translated_content(generator)
|
||
|
|
|
||
|
|
|
||
|
|
def get_pelican_cls(settings):
|
||
|
|
'''Get the Pelican class requested in settings'''
|
||
|
|
cls = settings['PELICAN_CLASS']
|
||
|
|
if isinstance(cls, six.string_types):
|
||
|
|
module, cls_name = cls.rsplit('.', 1)
|
||
|
|
module = __import__(module)
|
||
|
|
cls = getattr(module, cls_name)
|
||
|
|
return cls
|
||
|
|
|
||
|
|
|
||
|
|
def create_next_subsite(pelican_obj):
|
||
|
|
'''Create the next subsite using the lang-specific config
|
||
|
|
|
||
|
|
If there are no more subsites in the generation queue, update all
|
||
|
|
the generators (interlink translations and removed content, add
|
||
|
|
variables and translations to template context). Otherwise get the
|
||
|
|
language and overrides for next the subsite in the queue and apply
|
||
|
|
overrides. Then generate the subsite using a PELICAN_CLASS
|
||
|
|
instance and its run method. Finally, restore the previous locale.
|
||
|
|
'''
|
||
|
|
global _MAIN_SETTINGS
|
||
|
|
if len(_SUBSITE_QUEUE) == 0:
|
||
|
|
_LOGGER.debug(
|
||
|
|
'i18n: Updating cross-site links and context of all generators.')
|
||
|
|
update_generators()
|
||
|
|
_MAIN_SETTINGS = None # to initialize next time
|
||
|
|
else:
|
||
|
|
with temporary_locale():
|
||
|
|
settings = _MAIN_SETTINGS.copy()
|
||
|
|
lang, overrides = _SUBSITE_QUEUE.popitem()
|
||
|
|
settings.update(overrides)
|
||
|
|
settings = configure_settings(settings) # to set LOCALE, etc.
|
||
|
|
cls = get_pelican_cls(settings)
|
||
|
|
|
||
|
|
new_pelican_obj = cls(settings)
|
||
|
|
_LOGGER.debug(("Generating i18n subsite for language '{}' "
|
||
|
|
"using class {}").format(lang, cls))
|
||
|
|
new_pelican_obj.run()
|
||
|
|
|
||
|
|
|
||
|
|
# map: signal name -> function name
|
||
|
|
_SIGNAL_HANDLERS_DB = {
|
||
|
|
'get_generators': initialize_plugin,
|
||
|
|
'article_generator_pretaxonomy': filter_contents_translations,
|
||
|
|
'page_generator_finalized': filter_contents_translations,
|
||
|
|
'get_writer': create_next_subsite,
|
||
|
|
'static_generator_finalized': save_main_static_files,
|
||
|
|
'generator_init': save_generator,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def register():
|
||
|
|
'''Register the plugin only if required signals are available'''
|
||
|
|
for sig_name in _SIGNAL_HANDLERS_DB.keys():
|
||
|
|
if not hasattr(signals, sig_name):
|
||
|
|
_LOGGER.error((
|
||
|
|
'The i18n_subsites plugin requires the {} '
|
||
|
|
'signal available for sure in Pelican 3.4.0 and later, '
|
||
|
|
'plugin will not be used.').format(sig_name))
|
||
|
|
return
|
||
|
|
|
||
|
|
for sig_name, handler in _SIGNAL_HANDLERS_DB.items():
|
||
|
|
sig = getattr(signals, sig_name)
|
||
|
|
sig.connect(handler)
|