csv in and export, instances

This commit is contained in:
Alex 2018-06-10 12:15:07 +02:00
parent ab7a9c7ef6
commit 5ba0f10f13
24 changed files with 441 additions and 24 deletions

BIN
.DS_Store vendored

Binary file not shown.

View File

@ -40,7 +40,7 @@ def get_cover(file_path, filename):
big_filename = "app/uploads/cover/"+page["filename"] + "_cover.png"
small_filename = "app/uploads/cover/"+page["filename"] + "cover_small" + ".png"
img = pdf_page_to_png(src_pdf, pagenum = page["pagenum"], resolution = 300)
img = pdf_page_to_png(src_pdf, pagenum = page["pagenum"], resolution = 200)
img.save(filename = big_filename)
# Ensmallen

23
app/extractText.py Normal file
View File

@ -0,0 +1,23 @@
import PyPDF2
def get_text(file_path, filename):
read_pdf =file_path
with open(read_pdf,'rb') as pdf_file, open("app/uploads/"+filename+'.txt', 'w') as text_file:
read_pdf = PyPDF2.PdfFileReader(pdf_file)
number_of_pages = read_pdf.getNumPages()
for page_number in range(number_of_pages): # use xrange in Py2
page = read_pdf.getPage(page_number)
page_content = page.extractText()
text_file.write(page_content)
def extract_text(file_path, filename):
try:
get_text(file_path, filename)
except:
with open(filename+'.txt', 'w') as text_file:
page_content = ""
text_file.write(page_content)

View File

@ -15,6 +15,11 @@ stacks = db.Table('books_stacks',
db.Column('stack_id', db.Integer, db.ForeignKey('stacks.id'), primary_key=True)
)
instances = db.Table('books_instances',
db.Column('book_id', db.Integer, db.ForeignKey('books.id'), primary_key=True),
db.Column('instance_id', db.Integer, db.ForeignKey('instances.id'), primary_key=True)
)
class Book(db.Model):
__tablename__ = 'books'
__searchable__ = ['title', 'category', 'fileformat'] # these fields will be indexed by whoosh
@ -28,10 +33,13 @@ class Book(db.Model):
year_published = db.Column(db.Numeric(4,0))
description = db.Column(db.String(2500))
html = db.Column(db.String(255))
downloads = db.Column(db.Numeric(100,0))
authors = db.relationship('Author', secondary=authors,cascade="delete", lazy='subquery',
backref=db.backref('books', lazy=True),passive_deletes=True)
stacks = db.relationship('Stack', secondary=stacks, lazy='subquery',
backref=db.backref('books', lazy=True))
instances = db.relationship('Instance', secondary=instances, lazy='subquery',
backref=db.backref('books', lazy=True))
scapeX = db.Column(db.Numeric(10,2))
scapeY = db.Column(db.Numeric(10,2))
@ -42,6 +50,7 @@ class Book(db.Model):
self.fileformat = fileformat
self.category = category
self.year_published = year_published
self.download = None
self.scapeX = 0
self.scapeY = 0
@ -62,6 +71,19 @@ class Author(db.Model):
def __init__(self, author_name):
self.author_name = author_name
class Instance(db.Model):
__tablename__ = 'instances'
id = db.Column(db.Integer(), primary_key=True)
name = db.Column(db.String(50))
ip = db.Column(db.String(50))
action = db.Column(db.String(50))
def __init__(self, ip, action):
self.name = ip
self.ip = ip
self.action = action
class UserIns(db.Model):
__tablename__ = 'userins'

View File

@ -61,7 +61,7 @@ console.log(time)
return ('0'+time.getDate()).slice(-2) + '.' + ('0'+(time.getMonth()+1)).slice(-2) + '.' + time.getFullYear() +" " + ('0'+time.getHours()).slice(-2)+":"+ ('0'+time.getMinutes()).slice(-2);
}
//change addr when ONLINE::::::->
var socket = io.connect('http://localhost:5000');
var socket = io.connect('{{server}}');
var app = new Vue({
el: "#app",
delimiters: ['[[', ']]'],
@ -111,6 +111,11 @@ socket.on('channel-' + app.channel, function(msg) {
scrollTop: $('.messages')[0].scrollHeight
});
});
document.getElementById("file_import_csv").onchange = function() {
document.getElementById("form_import_csv").submit();
};
</script>
</body>
</html>

View File

@ -5,6 +5,7 @@
<li><a href="{{ url_for('show_stacks') }}">Stacks</a></li>
<li><a href="{{ url_for('add_book') }}">Add Book</a></li>
<li><a href="{{ url_for('about') }}">About</a></li>
<li><a href="{{ url_for('show_instances') }}">Instances</a></li>
</ul>
<div class="clearfix"></div>
</nav>

View File

@ -4,6 +4,8 @@
<div id="home_content">
<h1 class="header" id="title_xppl">XPPL</h1>
<p class="lead">This is the awesome library of Experimental Publishing. <br>
On instance: {{server}} / From: {{client}}
<br>
This might only be one interface to this library:
</p>
@ -12,6 +14,14 @@ This might only be one interface to this library:
<a href="{{url_for('show_books')}}">List</a>
<br><br><br>
<form method="GET" action="/export/csv">
<button type="submit">Export Catalogue (.CSV)</button>
</form>
<form method="POST" action="/import/csv" id="form_import_csv" enctype=multipart/form-data>
<input type="file" id="file_import_csv" name=file style="display:none;" />
<button type = "button" onclick="document.getElementById('file_import_csv').click()">Import Catalogue (.CSV)</button>
</form>
</div>
<div id="app" class="container">

View File

@ -0,0 +1,7 @@
{% extends "base.html" %}
{% block main %}
<h1 class="page-header">Imported CSV</h1>
<p>{{numberadded}} books added!</p>
{% endblock %}

View File

@ -5,25 +5,98 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" type="text/css" href="http://code.jquery.com/ui/1.9.2/themes/base/jquery-ui.css"/>
<style>
#scape_container {
zoom: 1;
}
body .ui-selecting { border:2px solid yellow; }
body .ui-selected {border:2px solid black;}
body {overflow: scroll;}
#scape_container{overflow: scroll;width: 100%; height:100vh;}
#zoombuttons{
position: fixed;
right:20px;
bottom: 20px;
z-index: +999999999999999999999999999999999999999999999;
user-select: none;
-moz-user-select: none;
-khtml-user-select: none;
-webkit-user-select: none;
-o-user-select: none;
}
#zoom-in{
width: 30px;
height: 30px;
background-color: rgb(240,240,240);
cursor: pointer;
border: 1px solid black;
text-align: center;
padding-top: 10px;
}
#zoom-out{
width: 30px;
height: 30px;
background-color: rgb(240,240,240);
cursor: pointer;
border: 1px solid black;
text-align: center;
padding-top: 10px;
}
#zoom-in:hover{
background-color: rgb(230,230,230);
}
#zoom-out:hover{
background-color: rgb(230,230,230);
}
.booktitle a{
text-decoration: none;
color: black;
}
</style>
</head>
<body>
<div id="zoombuttons">
<select name="selector" id="selector">
{% for instance in instances %}
<option value="{{instance.name}}">{{instance.name}}</option>
{% endfor %}
</select>
<div id="zoom-in">+</div>
<div id="zoom-out">-</div>
</div>
<div id ="scape_container">
{% for book in books|sort(attribute='title', reverse = False) %}
<div class = "drag" id = "{{ book.id }}" style="position: absolute;width:40px;height:auto; top:{{ book.scapeY }}px; left:{{ book.scapeX }}px;">
<div class = "drag" id = "{{ book.id }}" style="position: absolute;width:70px;height:auto; top:{{ book.scapeY }}px; left:{{ book.scapeX }}px;">
<img class="no_cover" id="{{ book.title }}" src="../uploads/cover/{{ book.cover }}" style="width:100%;height:auto;" onerror="if (this.src != '../static/img/default_cover.png') this.src = '../static/img/default_cover.png';">
<p style="font-size:7px;"><a href="books/{{ book.id }}">{{ book.title }}</a></p>
<p class="booktitle" style="font-size:7px;"><a href="books/{{ book.id }}">{{ book.title }}</a></p>
{% set got = {} %}
{% set all = 1 %}
{% for instance in book.instances %}
{% if instance.name in got %}
{% set x=got.__setitem__(instance.name, got[instance.name]+1) %}
{% else %}
{% set x=got.__setitem__(instance.name, 1) %}
{% endif %}
{% set all = loop.index %}
{% endfor %}
{% for instance, value in got.items() %}
{% set result = value/(book.instances|length) %}
<p hidden="True" >{{ instance }}</p>
<p hidden="True" class = "{{ instance }}" >{{result}}</p>
{% endfor %}
</div>
{% endfor %}
<div id="random" style="padding:2px; margin: 0;position: absolute;width:120px;height:20px;background-color:yellow;z-index:999999999999999999900000000000000;cursor:pointer;">
@ -81,7 +154,29 @@
<script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script>
<script>
$( function() {
var currentZoom = $('#scape_container').css('zoom');
var zoom = parseFloat(currentZoom);
var pointerX;
var pointerY;
$( ".drag" ).draggable({ stack: ".drag",
start : function(evt, ui) {
pointerY = (evt.pageY - $('#scape_container').offset().top) / zoom - parseInt($(evt.target).css('top'));
pointerX = (evt.pageX - $('#scape_container').offset().left) / zoom - parseInt($(evt.target).css('left'));
},
drag : function(evt, ui) {
var canvasTop = $('#scape_container').offset().top;
var canvasLeft = $('#scape_container').offset().left;
var canvasHeight = $('#scape_container').height();
var canvasWidth = $('#scape_container').width();
// Fix for zoom
ui.position.top = Math.round((evt.pageY - canvasTop) / zoom - pointerY);
ui.position.left = Math.round((evt.pageX - canvasLeft) / zoom - pointerX);
// Finally, make sure offset aligns with position
ui.offset.top = Math.round(ui.position.top + canvasTop);
ui.offset.left = Math.round(ui.position.left + canvasLeft);
},
stop: function(){
var offset = $(this).offset();
var id = $(this).attr('id');
@ -108,12 +203,7 @@
} );
$( function() {
$( ".drag" ).resizable({aspectRatio: true});
} );
/* $( function() {
$( "body" ).selectable();
} );*/
$( "#random" ).click(function() {
console.log("hallo");
@ -121,8 +211,8 @@ $( ".drag" ).each(function() {
var id = $(this).attr('id');
var postForm = { //Fetch form data
'id' : id,
'x' : Math.floor(Math.random() * window.innerWidth) , //Store name fields value,
'y' : Math.floor(Math.random() * window.innerHeight)
'x' : Math.floor(Math.random() * 2000) , //Store name fields value,
'y' : Math.floor(Math.random() * 2000)
};
$( this ).css("top", postForm['y']);
$( this ).css("left", postForm['x']);
@ -137,6 +227,42 @@ $( ".drag" ).each(function() {
});
});
$("#zoom-in").click(function(){
var currentZoom = $('#scape_container').css('zoom');
var newnumber = parseFloat(currentZoom) + 0.1;
console.log(newnumber)
//$('#scape_container').css('zoom','90%'); /* Webkit browsers */
$('#scape_container').css('zoom',newnumber.toString()); /* Other non-webkit browsers */
// $('#scape_container').css('-moz-transform',scale(0.9, 0.9)); /* Moz-browsers */
});
$("#zoom-out").click(function(){
var currentZoom = $('#scape_container').css('zoom');
var newnumber = parseFloat(currentZoom) - 0.1;
console.log(currentZoom+", "+newnumber.toString())
//$('#scape_container').css('zoom','90%'); /* Webkit browsers */
$('#scape_container').css('zoom',newnumber.toString()); /* Other non-webkit browsers */
// $('#scape_container').css('-moz-transform',scale(0.9, 0.9)); /* Moz-browsers */
});
$("select[name=selector]").on('change', function() {
var selectedInstance = $('select[name=selector]').val()
$( ".drag" ).each(function() {
if($(this).find('.'+selectedInstance).length==0)
{
$(this).css("opacity", "0.02")
}
else{
var found = $(this).find('.'+selectedInstance).html()
$(this).css("opacity", found)
}
});
});
</script>
</body>

View File

@ -21,6 +21,25 @@
<a href="../uploads/{{ book.file }}">download {{ book.fileformat }}</a>
<br>
<br>
<p>Instances:</p>
{% set got = {} %}
{% set all = 1 %}
{% for instance in book.instances %}
{% if instance.name in got %}
{% set x=got.__setitem__(instance.name, got[instance.name]+1) %}
{% else %}
{% set x=got.__setitem__(instance.name, 1) %}
{% endif %}
{% set all = loop.index %}
{% endfor %}
{% for instance, value in got.items() %}
{% set result = value/(book.instances|length) %}
{{ instance }}: {{ (result*100)|round|int }}%<br>
{% endfor %}
<br>
<br>
<a href="{{ url_for('edit_book_by_id', id=book.id )}}">edit</a>

View File

@ -0,0 +1,37 @@
{% extends 'base.html' %}
{% block main %}
<div class="container">
<h1 class="page-header">Instances</h1>
<p>All instances, that are using this library</p>
<table style="width:100%">
<!--
<td> {% for stack in stacks %}
<li><a href="{{url_for('show_stack_by_id', id=stack.id)}}">{{ stack.stack_name }}</a> </li>
{% endfor %}
</td>
</table>
-->
<ul>
{% for instance in instances %}
<li>{{ instance.name }} / {{ instance.ip }}</li>
<form method="post">
<input id="{{ instance.name }}" type="text" name="{{ instance.name }}"><br>
<input type="submit" value="rename">
</form>
{% endfor %}
</ul>
</div>
{% endblock %}

View File

@ -6,16 +6,20 @@ This file creates your application.
"""
from app import app, db, socketio, DOMAIN
from flask import Flask, render_template, request, redirect, url_for, flash, send_from_directory, jsonify, abort
from flask import Flask, Response, render_template, request, redirect, url_for, flash, send_from_directory, jsonify, abort
import json
from sqlalchemy.sql.expression import func, select
from app.forms import UploadForm, EditForm, SearchForm, ChatForm
from app.models import Book, BookSchema, Author, AuthorSchema, Stack, StackSchema, UserIns, Chat, ChatSchema
from app.models import Book, BookSchema, Author, AuthorSchema, Stack, StackSchema, UserIns, Chat, ChatSchema, Instance
from app.cover import get_cover
from app.extractText import extract_text
from os import environ
from flask_socketio import SocketIO, emit
import datetime
import time
from csv import DictWriter, DictReader
import io
from sqlalchemy.inspection import inspect
import os
from werkzeug.utils import secure_filename
@ -50,8 +54,14 @@ def home():
# msg = Chat(message)
# db.session.add(msg)
# db.session.commit()
#client = request.remote_addr
server = request.host
if request.environ.get('HTTP_X_FORWARDED_FOR') is None:
client =request.environ['REMOTE_ADDR']
else:
client = request.environ['HTTP_X_FORWARDED_FOR']
return render_template('home.html',domain=DOMAIN,chat=chat_messages, channel = 1, username="librarian")
return render_template('home.html',domain=DOMAIN,chat=chat_messages, channel = 1, username="librarian", client=client, server=server)
@app.route('/hello/<name>')
def hello(name):
@ -64,6 +74,13 @@ def about():
@app.route('/uploads/<filename>')
def uploaded_file(filename):
book = Book.query.filter_by(file=filename).first()
i = Instance(request.host, "download")
existing_ip = db.session.query(Instance).filter_by(ip=request.host).first()
if existing_ip:
i.name = existing_ip.name
book.instances.append(i)
db.session.commit()
return send_from_directory(app.config['UPLOAD_FOLDER'],
filename)
@ -90,7 +107,19 @@ def scape():
book.scapeY = data['y']
db.session.commit()
books = db.session.query(Book).all() # or you could have used User.query.all()
return render_template('scape.html', books=books)
all_instances = db.session.query(Instance).all()
instances = []
for instance in all_instances:
exists = False
for existing_inst in instances:
if existing_inst.name == instance.name:
exists = True
break
else:
exists = False
if not exists:
instances.append(instance)
return render_template('scape.html', books=books, instances=instances)
@app.route('/books_grid')
@ -101,6 +130,7 @@ def show_books_grid():
@app.route('/books/<int:id>')
def show_book_by_id(id):
book = Book.query.get(id)
all_instances = db.session.query(Instance).all()
userin = UserIns.query.filter_by(title="lastViewed").first()
if userin != None:
userin.info = book.title
@ -112,7 +142,7 @@ def show_book_by_id(id):
if not book:
return render_template('red_link.html', id=id)
else:
return render_template('show_book_detail.html', book=book)
return render_template('show_book_detail.html', book=book, all_instances=all_instances)
@app.route('/books/<int:id>/delete', methods=['POST', 'GET'])
@ -157,6 +187,11 @@ def edit_book_by_id(id):
a = Author(author_name=author_name)
db.session.add(a)
book.authors.append(a)
i = Instance(request.host, "edit")
existing_ip = db.session.query(Instance).filter_by(ip=request.host).first()
if existing_ip:
i.name = existing_ip.name
book.instances.append(i)
db.session.commit()
flash("%s updated" % (title))
return redirect(url_for('show_book_by_id', id=id))
@ -198,7 +233,13 @@ def add_book():
fullpath = os.path.join(app.config['UPLOAD_FOLDER'], new_filename)
name, file_extension = os.path.splitext(new_filename)
file.save(fullpath)
cover = get_cover(fullpath, name)
try:
cover = get_cover(fullpath, name)
except:
cover = ''
extract_text(fullpath, name)
else:
flash('allowed file formats: %s' % ALLOWED_EXTENSIONS)
#if upload without file -> wishform, with potential PDF
@ -221,6 +262,11 @@ def add_book():
a = Author(author_name=author_name)
db.session.add(a)
book.authors.append(a)
i = Instance(request.host, "add")
existing_ip = db.session.query(Instance).filter_by(ip=request.host).first()
if existing_ip:
i.name = existing_ip.name
book.instances.append(i)
db.session.commit()
flash("%s added to the library" % (title))
@ -269,7 +315,36 @@ def show_stack_by_id(id):
else:
return render_template('show_stack_detail.html', stack=stack)
## search
@app.route('/instances', methods=['POST', 'GET'])
def show_instances():
all_instances = db.session.query(Instance).all()
instances = []
for instance in all_instances:
exists = False
for existing_inst in instances:
if existing_inst.name == instance.name:
exists = True
break
else:
exists = False
if not exists:
instances.append(instance)
if request.method == 'POST':
for item in request.form.items():
for i, itm in enumerate(item):
if i == 0:
oldname = itm
if i == 1:
name = itm
all_instances = db.session.query(Instance).filter_by(name=oldname).all()
for instance in all_instances:
instance.name = name
print(oldname)
print(name)
db.session.commit()
return render_template('show_instances.html', instances=instances)
## search
@ -309,6 +384,96 @@ def search_results(searchtype, query):
return render_template('results.html', form=search, books=results, books_all=random_order, searchtype=search.select.data, query=query)
@app.route('/export/csv', methods=['GET'])
def export_csv():
output = io.StringIO()
#fieldnames = ['title', 'authors', 'file', 'fileformat', 'category', 'year_published', 'description' ]
fields = Book.__mapper__.columns
fieldnames = []
for columns in fields:
fieldnames.append(columns.name)
i = inspect(Book)
referred_classes = [r.mapper.class_ for r in i.relationships]
referred_classes_tablenames = [r.mapper.class_.__tablename__ for r in i.relationships]
print(fieldnames+referred_classes_tablenames)
csv = DictWriter(output,fieldnames+referred_classes_tablenames)
csv.writeheader()
for book in Book.query.order_by("title"):
row = {}
for col in fieldnames:
print(getattr(book, col))
row[col] = getattr(book, col)
for col in referred_classes:
subattr = []
for subcol in getattr(book, col.__tablename__):
for metacol in subcol.__mapper__.columns:
query = metacol.name
if query != "id":
this = getattr(subcol, query)
subattr.append(this)
row[col.__tablename__] = " | ".join(subattr)
csv.writerow(row)
#print(row)
resp = Response(output.getvalue(), mimetype="text/csv")
resp.headers["Content-Disposition"] = "attachment;filename=export.csv"
return resp
import codecs
@app.route('/import/csv', methods= ['POST','GET'])
def import_csv():
if request.method == 'POST':
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
else:
file = request.files['file']
for row in DictReader(codecs.iterdecode(file, 'utf-8')):
numberadded = 0;
book = Book.query.filter_by(title=row['title']).first()
if book:
print("allreadyexists")
else:
cover = ''
if row['file']:
fullpath = os.path.join(app.config['UPLOAD_FOLDER'], row['file'])
name, file_extension = os.path.splitext(row['file'])
print ('get_cover', fullpath, name)
cover = get_cover(fullpath, name)
if row['year_published']:
year_published = int(row['year_published'])
else:
year_published = None;
book = Book(row['title'], row['file'], cover, row['fileformat'], row['category'],year_published)
book.scapeX = float(row['scapeX'])
book.scapeY = float(row['scapeY'])
db.session.add(book)
numberadded = numberadded+1
authors = row['authors'].split('|')
authors = [x.strip() for x in authors]
for author in authors:
if author:
a = db.session.query(Author).filter_by(author_name=author).first()
if a == None:
a = Author(author_name=author)
db.session.add(a)
book.authors.append(a)
db.session.commit()
return render_template('import_csv.html', numberadded=numberadded)
@app.route('/empty_catalogue', methods= ['POST','GET'])
def empty_catalogue():
meta = db.metadata
for table in reversed(meta.sorted_tables):
if str(table) == "books" or str(table) == "authors" or str(table) == "books_authors":
print('Clear table %s' % table)
db.session.execute(table.delete())
db.create_all()
db.session.commit()
return "ALL CLEARED"
###
# The API
###
@ -384,7 +549,7 @@ def new_message(message):
my_new_chat = Chat(
message=message['text']
)
db.session.add(my_new_chat)
try:
db.session.commit()

View File

@ -4,4 +4,4 @@ mkdir -p app/uploads/cover
chmod 777 app/uploads/
chmod 777 app/uploads/cover
python3 init.py
python3 import_csv.py xpublibrary.csv
#python3 import_csv.py xpublibrary.csv

View File

@ -16,4 +16,6 @@ WTForms==2.1
flask-marshmallow==0.9.0
Wand==0.4.4
PyPDF2==1.26.0
flask-socketio==2.9.2
flask-whooshalchemyplus==0.7.5
python-dotenv==0.7.1

2
run.py
View File

@ -1,4 +1,4 @@
#! /usr/bin/env python
from app import app, socketio
socketio.run(app)
socketio.run(app,host="0.0.0.0", port=8080)
#app.run(debug=True,host="0.0.0.0",port=8080)

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
whoosh/Book/_MAIN_293.toc Normal file

Binary file not shown.