plugins: photos: videos: add PELICAN_SKIPTAG
Allows to tag output filenames with a given tag. Such a tag
is checked before processing new files with the result of
skipping files already tagged. Using SKIPTAG allows to feed
photos and videos using their related output directories and
being guaranteed not to process files multiple times.
clean_skiptag.sh is generated upon completion, if sourced it
removes original files from the gallery.
SKIPTAG was designed to allow:
INSTALLDIR_PHOTOS=PELICAN_PHOTO_GALLERY
and
INSTALLDIR_VIDEOS=PELICAN_VIDEO_GALLERY
diff --git a/pelicanconf.py b/pelicanconf.py
index d3b4035..e2b523b 100644
--- a/pelicanconf.py
+++ b/pelicanconf.py
@@ -29,6 +29,9 @@
_PATH = os.getenv('PELICAN_CONTENT')
PATH = _PATH
+# Where to output the generated files
+OUTPUT_PATH = 'output'
+
# Local path to the current theme folder
THEME = 'theme'
@@ -213,14 +216,17 @@
PHOTO_LIBRARY = os.getenv('PELICAN_PHOTO_LIBRARY')
PHOTO_EXCLUDE = os.getenv('PELICAN_PHOTO_EXCLUDE')
PHOTO_EXCLUDEALL = os.getenv('PELICAN_PHOTO_EXCLUDEALL')
+PHOTO_SKIPTAG = os.getenv('PELICAN_PHOTO_SKIPTAG')
PHOTO_GALLERY = (2000, 1333, 100)
PHOTO_ARTICLE = (2000, 1333, 100)
PHOTO_THUMB = (300, 200, 100)
PHOTO_SQUARE_THUMB = False
+PHOTO_RESIZE_JOBS = os.cpu_count()
# Videos plugin
VIDEO_LIBRARY = os.getenv('PELICAN_VIDEO_LIBRARY')
VIDEO_EXCLUDE = os.getenv('PELICAN_VIDEO_EXCLUDE')
VIDEO_EXCLUDEALL = os.getenv('PELICAN_VIDEO_EXCLUDEALL')
+VIDEO_SKIPTAG = os.getenv('PELICAN_VIDEO_SKIPTAG')
VIDEO_GALLERY = (720, 400, 100)
VIDEO_ARTICLE = (720, 400, 100)
diff --git a/plugins/photos/photos.py b/plugins/photos/photos.py
index 25027bc..1076cbd 100644
--- a/plugins/photos/photos.py
+++ b/plugins/photos/photos.py
@@ -64,6 +64,7 @@
DEFAULT_CONFIG.setdefault('PHOTO_LIGHTBOX_CAPTION_ATTR', 'data-title')
DEFAULT_CONFIG.setdefault('PHOTO_EXCLUDE', [])
DEFAULT_CONFIG.setdefault('PHOTO_EXCLUDEALL', False)
+ DEFAULT_CONFIG.setdefault('PHOTO_SKIPTAG', '')
DEFAULT_CONFIG['queue_resize'] = {}
DEFAULT_CONFIG['created_galleries'] = {}
@@ -96,6 +97,8 @@
pelican.settings['PHOTO_EXCLUDE'] = []
pelican.settings['PHOTO_EXCLUDEALL'] = pelican.settings['PHOTO_EXCLUDEALL'] == '1'
+ if pelican.settings['PHOTO_SKIPTAG'] is None:
+ pelican.settings.setdefault('PHOTO_SKIPTAG', '')
def read_notes(filename, msg=None):
notes = {}
@@ -257,6 +260,9 @@
return (img, piexif.dump(exif))
+# Define a global lock as 'apply_async' doesn't support sharing primitives
+GLock = multiprocessing.Lock()
+
def resize_worker(orig, resized, spec, settings):
logger.info('photos: make photo {} -> {}'.format(orig, resized))
im = Image.open(orig)
@@ -281,20 +287,47 @@
if isalpha(im):
im = remove_alpha(im, settings['PHOTO_ALPHA_BACKGROUND_COLOR'])
+ GLock.acquire()
if not os.path.exists(directory):
try:
os.makedirs(directory)
except Exception:
logger.exception('Could not create {}'.format(directory))
else:
- logger.debug('Directory already exists at {}'.format(os.path.split(resized)[0]))
+ if not settings['PHOTO_SKIPTAG']:
+ logger.debug('Directory already exists at {}'.format(os.path.split(resized)[0]))
+ GLock.release()
if settings['PHOTO_WATERMARK']:
isthumb = True if spec == settings['PHOTO_THUMB'] else False
if not isthumb or (isthumb and settings['PHOTO_WATERMARK_THUMB']):
im = watermark_photo(im, settings)
- im.save(resized, 'JPEG', quality=spec[2], icc_profile=icc_profile, exif=exif_copy)
+ try:
+ im.save(resized, 'JPEG', quality=spec[2], icc_profile=icc_profile, exif=exif_copy)
+
+ if settings['PHOTO_SKIPTAG']:
+ if not resized.endswith('.tmb.jpg'):
+ output_path = os.path.realpath(settings['OUTPUT_PATH'])
+ output_photos_path = os.path.join(output_path, 'photos')
+ cleaner_sh = os.path.join(output_photos_path, 'clean_skiptag.sh')
+ resized_relpath = os.path.relpath(resized, output_photos_path)
+ original_relpath = os.path.join(os.path.dirname(resized_relpath),
+ os.path.basename(orig))
+ try:
+ GLock.acquire()
+ with open(cleaner_sh, "a") as bash_script:
+ bash_line = "[ ! -f '{}' ] || rm -f '{}'"
+ print(bash_line.format(resized_relpath, original_relpath),
+ file=bash_script)
+ GLock.release()
+ except Exception as e:
+ except_msg = 'photos: could not open file {}'
+ logger.exception(except_msg.format(cleaner_sh))
+ GLock.release()
+
+ except Exception as e:
+ logger.exception('photos: could not save image {}'.format(resized))
def resize_photos(generator, writer):
@@ -349,15 +382,32 @@
)
if os.path.isfile(path):
+ original_filename = os.path.basename(path)
+ if original_filename.endswith('.tmb.jpg'):
+ return output
+
+ do_enqueue = True
+ do_rename = False
+ if settings['PHOTO_SKIPTAG']:
+ if original_filename.startswith(settings['PHOTO_SKIPTAG']):
+ debug_msg = 'photos: flagged to skip: {}'
+ logger.debug(debug_msg.format(original_filename))
+ do_enqueue = False
+ else:
+ do_rename = True
+
photo_prefix = os.path.splitext(value)[0].lower()
+ if do_rename:
+ photo_prefix = settings['PHOTO_SKIPTAG'] + photo_prefix
if what == 'photo':
- photo_article = photo_prefix + 'a.jpg'
- enqueue_resize(
- path,
- os.path.join('photos', photo_article),
- settings['PHOTO_ARTICLE']
- )
+ photo_article = photo_prefix + '.art.jpg'
+ if do_enqueue:
+ enqueue_resize(
+ path,
+ os.path.join('photos', photo_article),
+ settings['PHOTO_ARTICLE']
+ )
output = ''.join((
'<',
@@ -373,18 +423,19 @@
elif what == 'lightbox' and tag == 'img':
photo_gallery = photo_prefix + '.jpg'
- enqueue_resize(
- path,
- os.path.join('photos', photo_gallery),
- settings['PHOTO_GALLERY']
- )
+ if do_enqueue:
+ enqueue_resize(
+ path,
+ os.path.join('photos', photo_gallery),
+ settings['PHOTO_GALLERY']
+ )
- photo_thumb = photo_prefix + 't.jpg'
- enqueue_resize(
- path,
- os.path.join('photos', photo_thumb),
- settings['PHOTO_THUMB']
- )
+ photo_thumb = photo_prefix + '.tbm.jpg'
+ enqueue_resize(
+ path,
+ os.path.join('photos', photo_thumb),
+ settings['PHOTO_THUMB']
+ )
lightbox_attr_list = ['']
@@ -511,10 +562,38 @@
continue
if pic.endswith('.txt'):
continue
+ if pic.endswith('.tmb.jpg'):
+ continue
if pic in blacklist:
continue
+
+ do_enqueue = True
+ do_rename = False
+ if generator.settings['PHOTO_SKIPTAG']:
+ if pic.startswith(generator.settings['PHOTO_SKIPTAG']):
+ debug_msg = 'photos: flagged to skip: {}'
+ logger.debug(debug_msg.format(pic))
+ do_enqueue = False
+ else:
+ do_rename = True
+
photo = os.path.splitext(pic)[0].lower() + '.jpg'
- thumb = os.path.splitext(pic)[0].lower() + 't.jpg'
+ thumb = os.path.splitext(pic)[0].lower() + '.tmb.jpg'
+
+ if do_rename:
+ photo = generator.settings['PHOTO_SKIPTAG'] + photo
+ thumb = generator.settings['PHOTO_SKIPTAG'] + thumb
+
+ if do_enqueue:
+ enqueue_resize(
+ os.path.join(dir_gallery, pic),
+ os.path.join(dir_photo, photo),
+ generator.settings['PHOTO_GALLERY'])
+ enqueue_resize(
+ os.path.join(dir_gallery, pic),
+ os.path.join(dir_thumb, thumb),
+ generator.settings['PHOTO_THUMB'])
+
content_gallery.append((
pic,
os.path.join(dir_photo, photo),
@@ -522,15 +601,6 @@
exifs.get(pic, ''),
captions.get(pic, '')))
- enqueue_resize(
- os.path.join(dir_gallery, pic),
- os.path.join(dir_photo, photo),
- generator.settings['PHOTO_GALLERY'])
- enqueue_resize(
- os.path.join(dir_gallery, pic),
- os.path.join(dir_thumb, thumb),
- generator.settings['PHOTO_THUMB'])
-
content.photo_gallery.append((title, content_gallery))
logger.debug('Gallery Data: {}'.format(pprint.pformat(content.photo_gallery)))
DEFAULT_CONFIG['created_galleries']['gallery'] = content_gallery
@@ -564,20 +634,41 @@
image = file_clipper(image)
if os.path.isfile(path):
- photo = os.path.splitext(image)[0].lower() + 'a.jpg'
- thumb = os.path.splitext(image)[0].lower() + 't.jpg'
+ original_filename = os.path.basename(path)
+ if original_filename.endswith('.tmb.jpg'):
+ return
+
+ do_enqueue = True
+ do_rename = False
+ if generator.settings['PHOTO_SKIPTAG']:
+ if original_filename.startswith(generator.settings['PHOTO_SKIPTAG']):
+ debug_msg = 'photos: flagged to skip: {}'
+ logger.debug(debug_msg.format(original_filename))
+ do_enqueue = False
+ else:
+ do_rename = True
+
+ photo = os.path.splitext(image)[0].lower() + '.art.jpg'
+ thumb = os.path.splitext(image)[0].lower() + '.tmb.jpg'
+
+ if do_rename:
+ photo = generator.settings['PHOTO_SKIPTAG'] + photo
+ thumb = generator.settings['PHOTO_SKIPTAG'] + thumb
+
+ if do_enqueue:
+ enqueue_resize(
+ path,
+ os.path.join('photos', photo),
+ generator.settings['PHOTO_ARTICLE'])
+ enqueue_resize(
+ path,
+ os.path.join('photos', thumb),
+ generator.settings['PHOTO_THUMB'])
+
content.photo_image = (
os.path.basename(image).lower(),
os.path.join('photos', photo),
os.path.join('photos', thumb))
- enqueue_resize(
- path,
- os.path.join('photos', photo),
- generator.settings['PHOTO_ARTICLE'])
- enqueue_resize(
- path,
- os.path.join('photos', thumb),
- generator.settings['PHOTO_THUMB'])
else:
logger.error('photo: No photo for {} at {}'.format(content.source_path, path))
diff --git a/plugins/videos/videos.py b/plugins/videos/videos.py
index 67a27bc..fea7d71 100644
--- a/plugins/videos/videos.py
+++ b/plugins/videos/videos.py
@@ -266,6 +266,7 @@
DEFAULT_CONFIG.setdefault('VIDEO_CONVERT_JOBS', 1)
DEFAULT_CONFIG.setdefault('VIDEO_EXCLUDE', [])
DEFAULT_CONFIG.setdefault('VIDEO_EXCLUDEALL', False)
+ DEFAULT_CONFIG.setdefault('VIDEO_SKIPTAG', '')
DEFAULT_CONFIG['queue_convert'] = {}
DEFAULT_CONFIG['created_galleries'] = {}
@@ -288,6 +289,9 @@
video_excludeall = pelican.settings['VIDEO_EXCLUDEALL'] == '1'
pelican.settings['VIDEO_EXCLUDEALL'] = video_excludeall
+ if pelican.settings['VIDEO_SKIPTAG'] is None:
+ pelican.settings.setdefault('VIDEO_SKIPTAG', '')
+
def read_notes(filename, msg=None):
notes = {}
try:
@@ -326,17 +330,23 @@
license_line = 'Copyright {Year} {Author}, All Rights Reserved'
return license_line.format(Author = author, Year = year)
+# Define a global lock as 'apply_async' doesn't support sharing primitives
+GLock = multiprocessing.Lock()
+
def convert_worker(orig, converted, spec, settings):
directory = os.path.split(converted)[0]
+ GLock.acquire()
if not os.path.exists(directory):
try:
os.makedirs(directory)
except Exception:
logger.exception('videos: could not create {}'.format(directory))
else:
- debug_msg = 'videos: directory already exists at {}'
- logger.debug(debug_msg.format(os.path.split(converted)[0]))
+ if not settings['VIDEO_SKIPTAG']:
+ debug_msg = 'videos: directory already exists at {}'
+ logger.debug(debug_msg.format(os.path.split(converted)[0]))
+ GLock.release()
try:
c = Converter() # FIXME: is Converter thread safe?
@@ -345,6 +355,26 @@
for t in conv:
print(info_msg.format(t, '*' * int(t/2)), end='\r')
print(info_msg.format(100, '*' * int(t/2) + ' Done'), end='\n')
+
+ if settings['VIDEO_SKIPTAG']:
+ output_path = os.path.realpath(settings['OUTPUT_PATH'])
+ output_videos_path = os.path.join(output_path, 'videos')
+ cleaner_sh = os.path.join(output_videos_path, 'clean_skiptag.sh')
+ converted_relpath = os.path.relpath(converted, output_videos_path)
+ original_relpath = os.path.join(os.path.dirname(converted_relpath),
+ os.path.basename(orig))
+ try:
+ GLock.acquire()
+ with open(cleaner_sh, "a") as bash_script:
+ bash_line = "[ ! -f '{}' ] || rm -f '{}'"
+ print(bash_line.format(converted_relpath, original_relpath),
+ file=bash_script)
+ GLock.release()
+ except Exception as e:
+ except_msg = 'videos: could not open file {}'
+ logger.exception(except_msg.format(cleaner_sh))
+ GLock.release()
+
except Exception as e:
error_msg = 'videos: could not convert {} {}'
logger.exception(error_msg.format(orig, pprint.pformat(e)))
@@ -401,7 +431,7 @@
except Exception as e:
logger.exception('videos: none probed info {}'.format(pprint.pformat(e)))
-def enqueue_convert(orig, converted, input_vspec, settings):
+def enqueue_convert(orig, converted, input_vspec, settings, do_enqueue):
# Unfortunately need to compute width and height now, as templates
# need to be aware of the actual spec being computed.
try:
@@ -414,7 +444,8 @@
logger.exception(error_msg.format(orig, pprint.pformat(e)))
if converted not in DEFAULT_CONFIG['queue_convert']:
- DEFAULT_CONFIG['queue_convert'][converted] = (orig, output_spec)
+ if do_enqueue:
+ DEFAULT_CONFIG['queue_convert'][converted] = (orig, output_spec)
return (output_spec['video']['width'], output_spec['video']['height'])
if DEFAULT_CONFIG['queue_convert'][converted] != (orig, output_spec):
@@ -476,13 +507,26 @@
value)
if os.path.isfile(path):
+ original_filename = os.path.basename(path)
+ do_enqueue = True
+ do_rename = False
+ if settings['VIDEO_SKIPTAG']:
+ if original_filename.startswith(settings['VIDEO_SKIPTAG']):
+ debug_msg = 'videos: flagged to skip: {}'
+ logger.debug(debug_msg.format(original_filename))
+ do_enqueue = False
+ else:
+ do_rename = True
+
video_prefix = os.path.splitext(value)[0].lower()
+ if do_rename:
+ video_prefix = settings['VIDEO_SKIPTAG'] + video_prefix
if what == 'video':
ext = settings['VIDEO_FORMAT']
video_article = video_prefix + 'a.' + ext
enqueue_convert(path, os.path.join('videos', video_article),
- settings['VIDEO_ARTICLE'], settings)
+ settings['VIDEO_ARTICLE'], settings, do_enqueue)
output = ''.join((
'<',
@@ -594,16 +638,29 @@
continue
if old_video in blacklist:
continue
-
+
+ do_enqueue = True
+ do_rename = False
+ if generator.settings['VIDEO_SKIPTAG']:
+ if old_video.startswith(generator.settings['VIDEO_SKIPTAG']):
+ debug_msg = 'videos: flagged to skip: {}'
+ logger.debug(debug_msg.format(old_video))
+ do_enqueue = False
+ else:
+ do_rename = True
+
tag_format = generator.settings['VIDEO_FORMAT']
# tag_name currently not passed to the template
tag_name = os.path.splitext(old_video)[0].lower() + '.' + tag_format
+ if do_rename:
+ tag_name = generator.settings['VIDEO_SKIPTAG'] + tag_name
+
# relative path from /video/...
tag_path = os.path.join(dir_video, tag_name)
tag_spec = enqueue_convert(os.path.join(dir_gallery, old_video),
tag_path,
generator.settings['VIDEO_GALLERY'],
- generator.settings)
+ generator.settings, do_enqueue)
# Add more tag if needed, note: order matters!
content_gallery.append((tag_path, tag_spec[0], tag_spec[1], tag_format))
@@ -642,14 +699,29 @@
video_name = file_clipper(video_input)
if os.path.isfile(path):
+ original_filename = os.path.basename(path)
+
+ do_enqueue = True
+ do_rename = False
+ if generator.settings['VIDEO_SKIPTAG']:
+ if original_filename.startswith(generator.settings['VIDEO_SKIPTAG']):
+ debug_msg = 'videos: flagged to skip: {}'
+ logger.debug(debug_msg.format(original_filename))
+ do_enqueue = False
+ else:
+ do_rename = True
+
ext = generator.settings['VIDEO_FORMAT']
video = os.path.splitext(video_name)[0].lower() + 'a.' + ext
+ if do_rename:
+ video = generator.settings['VIDEO_SKIPTAG'] + video
+
content.video_file = (os.path.basename(video_name).lower(),
os.path.join('videos', video))
enqueue_convert(path, os.path.join('videos', video),
generator.settings['VIDEO_ARTICLE'],
- settings)
+ settings, do_enqueue)
else:
error_msg = 'videos: no video for {} at {}'
logger.error(error_msg.format(content.source_path, path))