Browse Source

Run the style checker over the source.

ansible-setup-and-deploy
Luke Murphy 6 years ago
parent
commit
5f4b5f3993
No known key found for this signature in database GPG Key ID: 5E2EF5A63E3718CC
  1. 19
      app/__init__.py
  2. 30
      app/cover.py
  3. 8
      app/extractText.py
  4. 48
      app/forms.py
  5. 50
      app/getannot.py
  6. 98
      app/models.py
  7. 0
      app/test.txt
  8. 666
      app/views.py

19
app/__init__.py

@ -1,14 +1,11 @@
import os
from os import environ
from dotenv import find_dotenv, load_dotenv
from flask import Flask from flask import Flask
from flask_socketio import SocketIO
from flask_sqlalchemy import SQLAlchemy from flask_sqlalchemy import SQLAlchemy
from marshmallow import Schema, fields, ValidationError, pre_load
from flask_socketio import SocketIO, emit
from os import environ
from dotenv import load_dotenv, find_dotenv
import os
import click
from werkzeug.utils import secure_filename
from sqlalchemy.dialects import registry from sqlalchemy.dialects import registry
# import flask_whooshalchemyplus not using whoosh anymore
registry.register("rqlite.pyrqlite", "sqlalchemy_rqlite.pyrqlite", "dialect") registry.register("rqlite.pyrqlite", "sqlalchemy_rqlite.pyrqlite", "dialect")
@ -16,23 +13,21 @@ basedir = os.path.abspath(os.path.dirname(__file__))
UPLOAD_FOLDER = os.path.join(basedir, 'uploads') UPLOAD_FOLDER = os.path.join(basedir, 'uploads')
UPLOAD_FOLDER_COVER = os.path.join(basedir, 'cover') UPLOAD_FOLDER_COVER = os.path.join(basedir, 'cover')
#ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
load_dotenv(find_dotenv()) load_dotenv(find_dotenv())
app = Flask(__name__) app = Flask(__name__)
app.config['SECRET_KEY'] = 'super secret key' app.config['SECRET_KEY'] = 'super secret key'
#app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/mydatabase.db'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['SQLALCHEMY_DATABASE_URI'] = 'rqlite+pyrqlite://localhost:4001/' app.config['SQLALCHEMY_DATABASE_URI'] = 'rqlite+pyrqlite://localhost:4001/'
app.config['DEBUG'] = True app.config['DEBUG'] = True
app.config['PORT'] = 80 app.config['PORT'] = 80
#app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(basedir, 'mydatabase.db')
db = SQLAlchemy(app) db = SQLAlchemy(app)
light = not os.path.isdir(UPLOAD_FOLDER) light = not os.path.isdir(UPLOAD_FOLDER)
DOMAIN = environ.get('DOMAIN') DOMAIN = environ.get('DOMAIN')
socketio = SocketIO(app) socketio = SocketIO(app)
app.config.from_object(__name__) app.config.from_object(__name__)
from app import views
from app import views # noqa

30
app/cover.py

@ -1,11 +1,11 @@
import io
import PyPDF2 import PyPDF2
from wand.image import Image
from wand.color import Color from wand.color import Color
import io from wand.image import Image
import os
def pdf_page_to_png(src_pdf, pagenum = 0, resolution = 72,): def pdf_page_to_png(src_pdf, pagenum=0, resolution=72):
""" """
Returns specified PDF page as wand.image.Image png. Returns specified PDF page as wand.image.Image png.
:param PyPDF2.PdfFileReader src_pdf: PDF from which to take pages. :param PyPDF2.PdfFileReader src_pdf: PDF from which to take pages.
@ -32,36 +32,32 @@ def get_cover(file_path, filename):
# ==== # ====
print(file_path) print(file_path)
src_filename = file_path src_filename = file_path
try: try:
src_pdf = PyPDF2.PdfFileReader(open(src_filename, "rb")) src_pdf = PyPDF2.PdfFileReader(open(src_filename, "rb"))
except: except Exception:
print("couln't open PDF") print("couln't open PDF")
return None; return
if src_pdf.isEncrypted: if src_pdf.isEncrypted:
try: try:
src_pdf.decrypt('') src_pdf.decrypt('')
except: except Exception:
print("couln't decrypt") print("couln't decrypt")
return None; return
# What follows is a lookup table of page numbers within sample_log.pdf and the corresponding filenames.
# What follows is a lookup table of page numbers within sample_log.pdf and
# the corresponding filenames.
pages = [{"pagenum": 0, "filename": filename}] pages = [{"pagenum": 0, "filename": filename}]
# Convert each page to a png image. # Convert each page to a png image.
for page in pages: for page in pages:
big_filename = "app/cover/"+page["filename"] + "_cover.jpeg" big_filename = "app/cover/"+page["filename"] + "_cover.jpeg"
#small_filename = "app/cover/"+page["filename"] + "cover_small" + ".jpeg"
img = pdf_page_to_png(src_pdf, pagenum=page["pagenum"], resolution=130) img = pdf_page_to_png(src_pdf, pagenum=page["pagenum"], resolution=130)
#img.save(filename = big_filename)
# Ensmallen # Ensmallen
img.transform("", "250") img.transform("", "250")
img.save(filename=big_filename) img.save(filename=big_filename)
return page["filename"] + "_cover.jpeg" return page["filename"] + "_cover.jpeg"
#---
#epub
#https://ebooks.stackexchange.com/questions/6517/command-line-extraction-of-metadata-title-author-from-epub-file
#https://hackage.haskell.org/package/epub-tools
#http://stackoverflow.com/questions/9751475/extract-cover-image-from-chm-and-epub-files

8
app/extractText.py

@ -3,21 +3,21 @@ import PyPDF2
def get_text(file_path, filename): def get_text(file_path, filename):
read_pdf = file_path read_pdf = file_path
write_txt = "app/uploads/" + filename + '.txt'
with open(read_pdf,'rb') as pdf_file, open("app/uploads/"+filename+'.txt', 'w') as text_file: with open(read_pdf, 'rb') as pdf_file, open(write_txt, 'w') as text_file:
read_pdf = PyPDF2.PdfFileReader(pdf_file) read_pdf = PyPDF2.PdfFileReader(pdf_file)
number_of_pages = read_pdf.getNumPages() number_of_pages = read_pdf.getNumPages()
for page_number in range(number_of_pages): # use xrange in Py2 for page_number in range(number_of_pages):
page = read_pdf.getPage(page_number) page = read_pdf.getPage(page_number)
page_content = page.extractText() page_content = page.extractText()
text_file.write(page_content) text_file.write(page_content)
def extract_text(file_path, filename): def extract_text(file_path, filename):
try: try:
get_text(file_path, filename) get_text(file_path, filename)
except: except Exception:
with open(filename + '.txt', 'w') as text_file: with open(filename + '.txt', 'w') as text_file:
page_content = "" page_content = ""
text_file.write(page_content) text_file.write(page_content)

48
app/forms.py

@ -1,11 +1,11 @@
from app.models import Author
from flask_wtf import FlaskForm from flask_wtf import FlaskForm
from wtforms import StringField, FileField, validators from wtforms import FieldList, FileField
from wtforms.validators import InputRequired, DataRequired
from wtforms import FieldList
from wtforms import Form as NoCsrfForm from wtforms import Form as NoCsrfForm
from wtforms.fields import StringField, FormField, SubmitField, SelectField, RadioField from wtforms import validators
from app.models import Book, BookSchema, Author, Stack, StackSchema from wtforms.fields import FormField, SelectField, StringField, SubmitField
from wtforms.fields.html5 import DecimalRangeField from wtforms.fields.html5 import DecimalRangeField
from wtforms.validators import DataRequired, InputRequired
# - - - Forms - - - # - - - Forms - - -
@ -13,11 +13,19 @@ class AuthorForm(NoCsrfForm):
# this forms is never exposed so we can user the non CSRF version # this forms is never exposed so we can user the non CSRF version
author_name = StringField('Author Name', validators=[DataRequired()]) author_name = StringField('Author Name', validators=[DataRequired()])
class UploadForm(FlaskForm): class UploadForm(FlaskForm):
title = StringField('title', validators=[InputRequired()]) title = StringField('title', validators=[InputRequired()])
author = FieldList(FormField(AuthorForm, default=lambda: Author()), min_entries=1) author = FieldList(
FormField(AuthorForm, default=lambda: Author()),
min_entries=1
)
category = StringField('category', validators=[InputRequired()]) category = StringField('category', validators=[InputRequired()])
year_published = StringField('year published', [validators.Length(max=4)],default=None) year_published = StringField(
'year published',
[validators.Length(max=4)],
default=None
)
file = FileField() file = FileField()
upload = SubmitField(label='Upload') upload = SubmitField(label='Upload')
wish = SubmitField(label='''I don't have the file, but wish I did.''') wish = SubmitField(label='''I don't have the file, but wish I did.''')
@ -37,9 +45,16 @@ class UploadForm(FlaskForm):
class EditForm(FlaskForm): class EditForm(FlaskForm):
title = StringField('title', validators=[InputRequired()]) title = StringField('title', validators=[InputRequired()])
author = FieldList(FormField(AuthorForm, default=lambda: Author()), min_entries=1) author = FieldList(
FormField(AuthorForm, default=lambda: Author()),
min_entries=1
)
category = StringField('category', validators=[InputRequired()]) category = StringField('category', validators=[InputRequired()])
year_published = StringField('year published', [validators.Length(max=4)],default=None) year_published = StringField(
'year published',
[validators.Length(max=4)],
default=None
)
file = FileField() file = FileField()
message = StringField('message') message = StringField('message')
sameness = DecimalRangeField('sameness', default=0) sameness = DecimalRangeField('sameness', default=0)
@ -54,22 +69,33 @@ class EditForm(FlaskForm):
('None of the above', 'None of the above')] ('None of the above', 'None of the above')]
who = SelectField('', choices=choices, default='Student') who = SelectField('', choices=choices, default='Student')
class ChatForm(FlaskForm): class ChatForm(FlaskForm):
message = StringField('message', validators=[InputRequired()]) message = StringField('message', validators=[InputRequired()])
send = SubmitField(label='Send') send = SubmitField(label='Send')
class StackForm(FlaskForm): class StackForm(FlaskForm):
stack_name = StringField('Stack', validators=[InputRequired()]) stack_name = StringField('Stack', validators=[InputRequired()])
stack_description = StringField('Description', validators=[InputRequired()]) stack_description = StringField(
'Description',
validators=[InputRequired()]
)
stack_author = StringField('Who made this', validators=[InputRequired()]) stack_author = StringField('Who made this', validators=[InputRequired()])
create = SubmitField(label='Create') create = SubmitField(label='Create')
class AddtoStackForm(FlaskForm): class AddtoStackForm(FlaskForm):
select_stack = SelectField('Stacks', validators=[InputRequired()]) select_stack = SelectField('Stacks', validators=[InputRequired()])
class EditStackForm(FlaskForm): class EditStackForm(FlaskForm):
edit_stack_name = StringField('Stack', validators=[InputRequired()]) edit_stack_name = StringField('Stack', validators=[InputRequired()])
edit_stack_description = StringField('Description', validators=[InputRequired()]) edit_stack_description = StringField(
'Description',
validators=[InputRequired()]
)
class SearchForm(FlaskForm): class SearchForm(FlaskForm):
choices = [('All', 'All'), choices = [('All', 'All'),

50
app/getannot.py

@ -1,8 +1,12 @@
from flask import request
import requests
import json import json
import requests
from flask import request
def get_annotations(): def get_annotations():
# TODO(decentral1se): remove this from the source
KEY = "6879-n8AksBoSB7kYoQ3eEwzpEr3nFQEmSp3XN-0PcKL_Sik" KEY = "6879-n8AksBoSB7kYoQ3eEwzpEr3nFQEmSp3XN-0PcKL_Sik"
# a dictionary containing necessary http headers # a dictionary containing necessary http headers
@ -17,11 +21,17 @@ def get_annotations():
search_url = "".join([base_url]) search_url = "".join([base_url])
r = requests.get(search_url, headers=headers) r = requests.get(search_url, headers=headers)
# data is a python dictionary # data is a python dictionary
data = json.loads(r.text) data = json.loads(r.text)
server = request.host server = request.host
for row in data['rows']: for row in data['rows']:
row['uri']= row['uri'].replace('http://' + server+'/uploads/','') row['uri'] = row['uri'].replace(
'http://' + server + '/uploads/',
''
)
return data return data
@ -31,15 +41,26 @@ def get_annot_results(annot,name):
for item in annot['rows']: for item in annot['rows']:
if 'selector' in item['target'][0]: if 'selector' in item['target'][0]:
if len(item['target'][0]['selector']) > 2: if len(item['target'][0]['selector']) > 2:
if name in item['text'] or name in item['target'][0]['selector'][2]['exact']: exact_second = item['target'][0]['selector'][2]['exact']
data={'text': item['text'],'extract':item['target'][0]['selector'][2]['exact'],'title':item['document']['title'], 'url':item['uri']} if name in item['text'] or name in exact_second:
data = {
'text': item['text'],
'extract': item['target'][0]['selector'][2]['exact'],
'title': item['document']['title'], 'url': item['uri']
}
res.append(data) res.append(data)
else: else:
if name in item['text'] or name in item['target'][0]['selector'][1]['exact']: exact_first = item['target'][0]['selector'][1]['exact']
data={'text': item['text'],'extract':item['target'][0]['selector'][1]['exact'],'title':item['document']['title'], 'url':item['uri']} if name in item['text'] or name in exact_first:
data = {
'text': item['text'],
'extract': item['target'][0]['selector'][1]['exact'],
'title': item['document']['title'], 'url': item['uri']
}
res.append(data) res.append(data)
return res return res
def get_annot_book(annot, name): def get_annot_book(annot, name):
res = [] res = []
server = request.host server = request.host
@ -48,14 +69,19 @@ def get_annot_book(annot,name):
if len(item['target'][0]['selector']) > 2: if len(item['target'][0]['selector']) > 2:
string = item['uri'] string = item['uri']
if name == string.replace('http://' + server+'/uploads/', ''): if name == string.replace('http://' + server+'/uploads/', ''):
data={'text': item['text'],'extract':item['target'][0]['selector'][2]['exact'],'title':item['document']['title'], 'url':item['uri']} data = {
'text': item['text'],
'extract': item['target'][0]['selector'][2]['exact'],
'title': item['document']['title'], 'url': item['uri']
}
res.append(data) res.append(data)
else: else:
string = item['uri'] string = item['uri']
if name == string.replace('http://' + server+'/uploads/', ''): if name == string.replace('http://' + server+'/uploads/', ''):
data={'text': item['text'],'extract':item['target'][0]['selector'][1]['exact'],'title':item['document']['title'], 'url':item['uri']} data = {
'text': item['text'],
'extract': item['target'][0]['selector'][1]['exact'],
'title': item['document']['title'], 'url': item['uri']
}
res.append(data) res.append(data)
return res return res

98
app/models.py

@ -1,24 +1,58 @@
from app import db
from marshmallow import Schema, fields, ValidationError, pre_load
import datetime import datetime
from sqlalchemy import Column, Integer, DateTime
import flask_whooshalchemyplus
authors = db.Table('books_authors', from app import db
db.Column('book_id', db.Integer, db.ForeignKey('books.id'), primary_key=True), from marshmallow import Schema, ValidationError, fields
db.Column('author_id', db.Integer, db.ForeignKey('authors.id'), primary_key=True) from sqlalchemy import Column, DateTime
authors = db.Table(
'books_authors',
db.Column(
'book_id',
db.Integer,
db.ForeignKey('books.id'),
primary_key=True
),
db.Column(
'author_id',
db.Integer,
db.ForeignKey('authors.id'),
primary_key=True
)
) )
stacks = db.Table('books_stacks', stacks = db.Table(
db.Column('book_id', db.Integer, db.ForeignKey('books.id'), primary_key=True), 'books_stacks',
db.Column('stack_id', db.Integer, db.ForeignKey('stacks.id'), primary_key=True) db.Column(
'book_id',
db.Integer,
db.ForeignKey('books.id'),
primary_key=True
),
db.Column(
'stack_id',
db.Integer,
db.ForeignKey('stacks.id'),
primary_key=True
)
) )
instances = db.Table('books_instances', instances = db.Table(
db.Column('book_id', db.Integer, db.ForeignKey('books.id'), primary_key=True), 'books_instances',
db.Column('instance_id', db.Integer, db.ForeignKey('instances.id'), primary_key=True) db.Column(
'book_id',
db.Integer,
db.ForeignKey('books.id'),
primary_key=True
),
db.Column(
'instance_id',
db.Integer,
db.ForeignKey('instances.id'),
primary_key=True
) )
)
class Book(db.Model): class Book(db.Model):
__tablename__ = 'books' __tablename__ = 'books'
@ -33,12 +67,22 @@ class Book(db.Model):
description = db.Column(db.String(2500)) description = db.Column(db.String(2500))
html = db.Column(db.String(255)) html = db.Column(db.String(255))
downloads = db.Column(db.Numeric(100, 0)) downloads = db.Column(db.Numeric(100, 0))
authors = db.relationship('Author', secondary=authors,cascade="delete", lazy='subquery', authors = db.relationship(
backref=db.backref('books', lazy=True),passive_deletes=True) 'Author',
stacks = db.relationship('Stack', secondary=stacks, lazy='subquery', secondary=authors,
backref=db.backref('books', lazy=True)) cascade="delete",
instances = db.relationship('Instance', secondary=instances, lazy='subquery', lazy='subquery',
backref=db.backref('books', lazy=True)) backref=db.backref('books', lazy=True),
passive_deletes=True
)
stacks = db.relationship(
'Stack', secondary=stacks, lazy='subquery',
backref=db.backref('books', lazy=True)
)
instances = db.relationship(
'Instance', secondary=instances, lazy='subquery',
backref=db.backref('books', lazy=True)
)
scapeX = db.Column(db.Numeric(10, 2)) scapeX = db.Column(db.Numeric(10, 2))
scapeY = db.Column(db.Numeric(10, 2)) scapeY = db.Column(db.Numeric(10, 2))
message = db.Column(db.String(1000)) message = db.Column(db.String(1000))
@ -47,8 +91,9 @@ class Book(db.Model):
gender = db.Column(db.Numeric()) gender = db.Column(db.Numeric())
who = db.Column(db.String(255)) who = db.Column(db.String(255))
def __init__(self, title, file, cover, fileformat,
def __init__(self, title, file, cover, fileformat, category, year_published, message, sameness, diversity, gender, who): category, year_published, message,
sameness, diversity, gender, who):
self.title = title self.title = title
self.file = file self.file = file
self.cover = cover self.cover = cover
@ -64,7 +109,6 @@ class Book(db.Model):
self.gender = gender self.gender = gender
self.who = who self.who = who
def __repr__(self): def __repr__(self):
return '<Title %r>' % self.title return '<Title %r>' % self.title
@ -81,6 +125,7 @@ class Author(db.Model):
def __init__(self, author_name): def __init__(self, author_name):
self.author_name = author_name self.author_name = author_name
class Instance(db.Model): class Instance(db.Model):
__tablename__ = 'instances' __tablename__ = 'instances'
@ -106,6 +151,7 @@ class UserIns(db.Model):
self.title = title self.title = title
self.info = info self.info = info
class Chat(db.Model): class Chat(db.Model):
__tablename__ = 'chat' __tablename__ = 'chat'
@ -117,14 +163,15 @@ class Chat(db.Model):
self.message = message self.message = message
self.time = datetime.datetime.now() self.time = datetime.datetime.now()
class Stack(db.Model): class Stack(db.Model):
__tablename__ = 'stacks' __tablename__ = 'stacks'
id = db.Column(db.Integer, primary_key=True) id = db.Column(db.Integer, primary_key=True)
stack_name = db.Column(db.String(50)) stack_name = db.Column(db.String(50))
stack_description = db.Column(db.String(1000)) stack_description = db.Column(db.String(1000))
stack_author = db.Column(db.String(255)) stack_author = db.Column(db.String(255))
def __init__(self, stack_name, stack_description, stack_author): def __init__(self, stack_name, stack_description, stack_author):
self.stack_name = stack_name self.stack_name = stack_name
self.stack_description = stack_description self.stack_description = stack_description
@ -149,16 +196,19 @@ class AuthorSchema(Schema):
id = fields.Int(dump_only=True) id = fields.Int(dump_only=True)
author_name = fields.Str() author_name = fields.Str()
class StackSchema(Schema): class StackSchema(Schema):
id = fields.Int(dump_only=True) id = fields.Int(dump_only=True)
stack_name = fields.Str() stack_name = fields.Str()
stack_description = fields.Str() stack_description = fields.Str()
class ChatSchema(Schema): class ChatSchema(Schema):
id = fields.Int(dump_only=True) id = fields.Int(dump_only=True)
message = fields.Str() message = fields.Str()
time = fields.DateTime() time = fields.DateTime()
class BookSchema(Schema): class BookSchema(Schema):
id = fields.Int(dump_only=True) id = fields.Int(dump_only=True)
title = fields.Str() title = fields.Str()

0
app/test.txt

666
app/views.py

File diff suppressed because it is too large
Loading…
Cancel
Save