Skip to content
Snippets Groups Projects
Commit ac4bbd40 authored by zshan2's avatar zshan2
Browse files

assignment-2.2 ver3.0: basic functions loaded

parent bc96133c
No related branches found
No related tags found
1 merge request!2Implementation of assignment-2.2
Hide whitespace changes
Inline Side-by-side
DataBase/mongoDB.py
+ 12
− 12
View file @ ac4bbd40
......@@ -16,9 +16,9 @@ def get_db():
def insert_document(docu, opt):
db = get_db()
if opt == 0:
records = db.test_books
records = db.books
elif opt == 1:
records = db.test_authors
records = db.authors
else:
print("failed to get json file: wrong opt for selecting collection")
return
......@@ -34,9 +34,9 @@ def insert_dicts(dictionary, opt):
"""
db = get_db()
if opt == 0:
records = db.test_books
records = db.books
elif opt == 1:
records = db.test_authors
records = db.authors
else:
print("failed to get json file: wrong opt for selecting collection")
return
......@@ -58,9 +58,9 @@ def update_dicts(opt, identifier, content):
"""
db = get_db()
if opt == 0:
records = db.test_books
records = db.books
elif opt == 1:
records = db.test_authors
records = db.authors
else:
print("failed to get json file: wrong opt for selecting collection")
return
......@@ -82,9 +82,9 @@ def get_documents_json(opt, identifier):
"""
db = get_db()
if opt == 0:
records = db.test_books
records = db.books
elif opt == 1:
records = db.test_authors
records = db.authors
else:
print("failed to get json file: wrong opt for selecting collection")
return json.dumps({})
......@@ -127,10 +127,10 @@ def clean(opt, identifier):
"""
db = get_db()
if opt == 0:
records = db.test_books
records = db.books
elif opt == 1:
records = db.test_authors
records = db.authors
else:
print("failed to get json file: wrong opt for selecting collection")
return
records.delete_many(identifier)
return 0
return records.delete_many(identifier).deleted_count
ManualTest.md 0 → 100644
+ 0
− 0
View file @ ac4bbd40
Server/SimpleServer.py
+ 110
− 70
View file @ ac4bbd40
......@@ -2,106 +2,144 @@ import DataBase.mongoDB as DataBase
import RegularExpressionParser.Parser as Parser
import re
import Crawler.Scrape as Scrape
import json
from flask import Flask
from flask import request
from flask import abort
from flask import jsonify
from urllib.parse import urlparse, parse_qs
from flask_cors import CORS
from flask_cors import CORS, cross_origin
app = Flask(__name__)
cors = CORS(app)
# app.config["DEBUG"] = True
# cors = CORS(app, resources={r"/api/*": {"origins": "*"}})
@app.route("/", methods=['GET'])
def home():
""" homepage of server """
return "200: successfully connected to home page\n"
@app.route("/api/<collection>/", methods=["GET", "PUT", "POST", "DELETE"])
def data_base(collection):
@app.route("/api/book/", methods=["GET", "PUT", "POST", "DELETE", "OPTIONS"])
def data_base_book():
print(request.url)
""" data base page of server """
if request.method == "GET":
if collection == "book":
url_parsed = urlparse(request.url)
qs_parsed = parse_qs(url_parsed.query)
if qs_parsed == {}:
return DataBase.get_documents_json(0, {})
result = search_document(["book.id:" + qs_parsed["id"][0]])
if result == {"wrong": "True"}:
abort(400, "Bad Request")
else:
return result
elif collection == "author":
url_parsed = urlparse(request.url)
qs_parsed = parse_qs(url_parsed.query)
if qs_parsed == {}:
return DataBase.get_documents_json(1, {})
result = search_document(["author.id:" + qs_parsed["id"][0]])
if result == {"wrong": "True"}:
abort(400, "Bad Request")
else:
return result
elif collection == "search":
url_parsed = urlparse(request.url)
query = Parser.parse_url_to_query(url_parsed.query)
qs_parsed = query.replace("q=", "")
result = search_document(qs_parsed.split("&"))
if result == {"wrong": "True"}:
abort(400, "Bad Request")
else:
return result
else:
url_parsed = urlparse(request.url)
qs_parsed = parse_qs(url_parsed.query)
if qs_parsed == {}:
return DataBase.get_documents_json(0, {})
result = search_document(["book.id:" + qs_parsed["id"][0]])
if result == {"wrong": "True"}:
abort(400, "Bad Request")
elif result == json.dumps({'books': []}):
abort(400, "Book not found")
else:
return result
elif request.method == "PUT":
if request.headers["Content-Type"] != "application/json":
abort(415)
abort(415, "content should be JSON file")
print(request)
json_update_info = request.json
if collection == "book":
opt = 0
elif collection == "author":
opt = 1
else:
abort(400, "Bad Request")
print(request)
print(json_update_info)
print(request.args.to_dict())
opt = 0
DataBase.update_dicts(opt, request.args.to_dict(), json_update_info)
return "200: PUT succeeded"
elif request.method == "POST":
if request.headers["Content-Type"] != "application/json":
abort(415, "content should be JSON file")
print(request)
json_file = request.json
if collection == "books":
DataBase.insert_dicts(json_file, 0)
elif collection == "authors":
DataBase.insert_dicts(json_file, 1)
elif collection == "book":
DataBase.insert_document(json_file, 0)
elif collection == "author":
DataBase.insert_document(json_file, 1)
elif collection == "scrape":
param = request.args.to_dict()
url = param["url"]
max_book = param["max_book"]
max_author = param["max_author"]
Scrape.scrape_api(url, max_book, max_author)
return "200: new data has been added to database"
print(request)
print(json_file)
DataBase.insert_document(json_file, 0)
return "200: POST succeeded"
elif request.method == "DELETE":
identifier = request.args.to_dict()
print(identifier)
opt=0
deleted_count = DataBase.clean(opt, identifier)
if deleted_count > 0:
return "200: DELETE succeeded"
else:
abort(400, "Failed to delete: target not found")
elif request.method == "OPTIONS":
return '200 OK'
@app.route("/api/author/", methods=["GET", "PUT", "POST", "DELETE", "OPTIONS"])
def data_base_author():
print(request.url)
""" data base page of server """
if request.method == "GET":
url_parsed = urlparse(request.url)
qs_parsed = parse_qs(url_parsed.query)
if qs_parsed == {}:
return DataBase.get_documents_json(1, {})
result = search_document(["author.id:" + qs_parsed["id"][0]])
if result == {"wrong": "True"}:
abort(400, "Bad Request")
elif result == json.dumps({'authors': []}):
abort(400, "Author not found")
else:
return result
elif request.method == "PUT":
if request.headers["Content-Type"] != "application/json":
abort(415, "content should be JSON file")
json_update_info = request.json
opt = 1
DataBase.update_dicts(opt, request.args.to_dict(), json_update_info)
return "200: PUT succeeded"
elif request.method == "POST":
if request.headers["Content-Type"] != "application/json":
abort(415, "content should be JSON file")
json_file = request.json
DataBase.insert_document(json_file, 1)
return "200: POST succeeded"
elif request.method == "DELETE":
identifier = request.args.to_dict()
print(identifier)
if collection == "book":
opt = 0
elif collection == "author":
opt = 1
opt=1
deleted_count = DataBase.clean(opt, identifier)
if deleted_count > 0:
return "200: DELETE succeeded"
else:
abort(400, "Unknown Collection to DELETE") # 400 cz 404 is for non-existing directory!
DataBase.clean(opt, identifier)
return "200: DELETE succeeded"
abort(400, "Failed to delete: target not found")
elif request.method == "OPTIONS":
return '200 OK'
@app.route("/api/scrape/", methods=["GET", "PUT", "POST", "DELETE", "OPTIONS"])
def data_base_scrape():
print(request.url)
param = request.args.to_dict()
print(param)
url = param["url"]
max_book = param["max_book"]
max_author = param["max_author"]
Scrape.scrape_api(url, max_book, max_author)
return "200: new data has been added to database"
@app.route("/api/search/", methods=["GET", "PUT", "POST", "DELETE", "OPTIONS"])
def data_base_search():
print(request.url)
url_parsed = urlparse(request.url)
query = Parser.parse_url_to_query(url_parsed.query)
qs_parsed = query.replace("q=", "")
result = search_document(qs_parsed.split("&"))
if result == {"wrong": "True"}:
abort(400, "Bad Request")
elif result == json.dumps({'books': []}):
abort(400, "Target not found")
else:
return result
# @app.route("/api/", methods=["GET", "PUT", "POST", "DELETE", "OPTIONS"])
# def data_api():
# print('====================')
# print(request.url)
# print(request.collection)
# print('====================')
# return '200: nice!'
def search_document(identifiers):
""" function used to find one or several document in database """
if len(identifiers) == 1:
......@@ -145,5 +183,7 @@ def search_document(identifiers):
return {}
if __name__ == "__main__":
app.run()
web/app/src/App.js
+ 239
− 61
View file @ ac4bbd40
//import logo from './logo.svg';
//import './App.css';
//
//function App() {
// return (
// <div className="App">
// <header className="App-header">
// <img src={logo} className="App-logo" alt="logo" />
// <p>
// Edit <code>src/App.js</code> and save to reload.
// </p>
// <a
// className="App-link"
// href="https://reactjs.org"
// target="_blank"
// rel="noopener noreferrer"
// >
// Learn React
// </a>
// </header>
// </div>
// );
//}
//
//export default App;
import React, { useState } from 'react';
import React from 'react';
import FourButtons from './Components/FourButtons';
import { Dialog } from 'react-overlay-pack';
import CorrectSign from './material/sign_correct.png';
import ErrorSign from './material/sign_error.png';
import ElementDisplay from './Components/ElementDisplay';
function App() {
const { useState } = React;
const [queryStr, setQueryStr] = useState('')
const [sign, setSign] = useState(CorrectSign)
const [text, setText] = useState('This area shows the result of requests..')
const [formState, setFormState] = useState('hide')
const [dialogState, setDialogState] = useState(false)
const [renderState, setRenderState] = useState(false)
const [bookURLState, setBookURLState] = useState('')
const [bookTitleState, setBookTitleState] = useState('')
const [bookIDState, setBookIDState] = useState('')
const [bookISBNState, setBookISBNState] = useState('')
const [bookAuthorState, setBookAuthorState] = useState('')
const [bookAuthorURLState, setBookAuthorURLState] = useState('')
const [bookRatingState, setBookRatingState] = useState('')
const [bookRatingCountState, setBookRatingCountState] = useState('')
const [bookReviewCountState, setBookReviewCountState] = useState('')
const [bookImageURLState, setBookImageURLState] = useState('')
const [bookSimilarBooksState, setBookSimilarBooksState] = useState('')
const [authorNameState, setAuthorNameState] = useState('')
const [authorURLState, setAuthorURLState] = useState('')
const [authorIDState, setAuthorIDState] = useState('')
const [authorRatingState, setAuthorRatingState] = useState('')
const [authorRatingCountState, setAuthorRatingCountState] = useState('')
const [authorReviewCountState, setAuthorReviewCountState] = useState('')
const [authorImageURLState, setAuthorImageURLState] = useState('')
const [authorRelatedAuthorsState, setAuthorRelatedAuthorsState] = useState('')
const [authorAuthorBooksState, setAuthorAuthorBooksState] = useState('')
const [dataState, setDataState] = useState({})
const axios = require('axios').default;
var center = {display: 'flex', justifyContent: 'center', alignItems: 'center'}
function get() {
axios.get('http://127.0.0.1:5000/api/' + queryStr)
.then(function (response) {
......@@ -48,24 +51,69 @@ function App() {
setSign(CorrectSign)
showDialog()
document.getElementById('dialog').value = response.status + ":\n" + response.statusText
document.getElementById('board').value = JSON.stringify(response.data)
if (queryStr.includes('book')) {
setDataState(response.data['books'][0])
} else {
setDataState(response.data['authors'][0])
}
setRenderState(true)
})
.catch(function (error) {
// handle error
setSign(ErrorSign)
showDialog()
document.getElementById('dialog').value = error
document.getElementById('dialog').value = error //.response.status + ":\n" + error.response.statusText
})
}
function put() {
changeFormState()
function createJson() {
var bookData = {}
bookData.type = 'book'
if(bookURLState !== '') {bookData.book_url = bookURLState}
if(bookTitleState !== '') {bookData.title = bookTitleState}
if(bookIDState !== '') {bookData.id = bookIDState}
if(bookISBNState !== '') {bookData.ISBN = bookISBNState}
if(bookAuthorURLState !== '') {bookData.author_url = bookAuthorURLState}
if(bookAuthorState !== '') {bookData.author = bookAuthorState}
if(bookRatingState !== '') {bookData.rating = parseFloat(bookRatingState)}
if(bookRatingCountState !== '') {bookData.rating_count = parseInt(bookRatingCountState)}
if(bookReviewCountState !== '') {bookData.review_count = parseInt(bookReviewCountState)}
if(bookSimilarBooksState !== '') {bookData.similar_books = bookSimilarBooksState.split(', ')}
setDataState(bookData)
return bookData
}
function createJsonAuthor() {
var authorData = {}
authorData.type = 'author'
if(authorNameState !== '') {authorData.name = authorNameState}
if(authorURLState !== '') {authorData.author_url = authorURLState}
if(authorIDState !== '') {authorData.id = authorIDState}
if(authorRatingState !== '') {authorData.rating = parseFloat(authorRatingState)}
if(authorRatingCountState !== '') {authorData.rating_count = parseInt(authorRatingCountState)}
if(authorReviewCountState !== '') {authorData.review_count = parseInt(authorReviewCountState)}
if(authorImageURLState !== '') {authorData.image_url = authorImageURLState}
if(authorRelatedAuthorsState !== '') {authorData.related_authors = authorRelatedAuthorsState.split(', ')}
if(authorAuthorBooksState !== '') {authorData.author_books = authorAuthorBooksState.split(',')}
setDataState(authorData)
return authorData
}
function hideForm() {
setFormState('hide')
}
function mesg() {
document.getElementById('board').value = 'Hello there!'
function changeFormStateBook() {
setRenderState(false)
if (formState === 'hide') {
setFormState('showBook')
} else {
setFormState('hide')
}
}
function changeFormState() {
function changeFormStateAuthor() {
setRenderState(false)
if (formState === 'hide') {
setFormState('show')
setFormState('showAuthor')
} else {
setFormState('hide')
}
......@@ -73,17 +121,91 @@ function App() {
function showDialog() {
setDialogState(true)
}
function put() {
setRenderState(false)
hideForm()
var data;
if (queryStr === 'book') {
data = createJson()
} else {
data = createJsonAuthor()
}
let config = {
headers: {
'Content-Type': 'application/json',
}
}
axios.put('http://127.0.0.1:5000/api/' + queryStr, JSON.stringify(data), config)
.then(function (response) {
// handle success
setSign(CorrectSign)
showDialog()
document.getElementById('dialog').value = response.status + ":\n" + response.statusText
})
.catch(function (error) {
// handle error
setSign(ErrorSign)
showDialog()
document.getElementById('dialog').value = error //.response.status + ":\n" + error.response.statusText
})
}
function post() {
setRenderState(false)
let config = {
headers: {
'Content-Type': 'application/json',
}
}
hideForm()
var data;
if (queryStr === 'book') {
data = createJson()
} else {
data = createJsonAuthor()
}
axios.post('http://127.0.0.1:5000/api/' + queryStr, JSON.stringify(data), config)
.then(function (response) {
// handle success
setSign(CorrectSign)
showDialog()
document.getElementById('dialog').value = response.status + ":\n" + response.statusText
})
.catch(function (error) {
// handle error
setSign(ErrorSign);
showDialog();
document.getElementById('dialog').value = error //.response.status + ":\n" + error.response.statusText
})
}
function delete_data() {
setRenderState(false)
axios.delete('http://127.0.0.1:5000/api/' + queryStr)
.then(function (response) {
// handle success
setSign(CorrectSign)
showDialog()
document.getElementById('dialog').value = response.status + ":\n" + response.statusText
})
.catch(function (error) {
// handle error
setSign(ErrorSign)
showDialog()
document.getElementById('dialog').value = error //.response.status + ":\n" + error.response.statusText
})
}
return (
<div>
{dialogState === true &&
<Dialog
<Dialog //Warning, not responsive!
show={dialogState}
onOutsideClick={() => setDialogState(false)}>
<div style={{marginTop:'20%'}}>
<div style={{
marginLeft: '35%'
}}>
<img src={sign}/>
<img alt='responseStatus' src={sign}/>
</div>
<div>
<textarea id='dialog'
......@@ -101,7 +223,6 @@ function App() {
</div>
</div>
</Dialog>}
<form name='mainform'>
<h1 style={center}>
Welcome to the home page of GoodReads Crawler! </h1>
<h3 style={center}> Please input your query string:</h3>
......@@ -109,7 +230,7 @@ function App() {
<input
id='queryString'
type='text'
placeholder='example: book?id=12345678'
placeholder='example: book/?id=12345678'
size='40'
value={queryStr}
onChange={(e) => setQueryStr(e.target.value)}
......@@ -118,27 +239,84 @@ function App() {
<h3 style={center}> Please input your data for uploading (only effective for POST and PUT):</h3>
{formState === 'hide' &&
<div style={center}>
<button style={{width:60}} onClick={changeFormState}>
<button style={{width:60}} onClick={changeFormStateBook}>
book
</button>
<button style={{marginLeft:10, width:60}} onClick={changeFormState}>
<button style={{marginLeft:10, width:60}} onClick={changeFormStateAuthor}>
author
</button>
</div>}
{formState === 'show' &&
<form style={center}>
<table border="1" style="width:100%;">
<tr>
<td>Cell 1</td>
<input/>
</tr>
<tr>
<td>Cell 3</td>
<input/>
</tr>
</table>
</form>}
</form>
{formState === 'showBook' &&
<form style={{border:'2px solid silver', marginLeft:'180px', marginRight: '180px'}}>
<div style={center}>
<input id='bookURL' style={center} type='text' size='40' value={bookURLState} placeholder='bookURL' onChange={(e) => setBookURLState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookTtile' style={center} type='text' size='40' value={bookTitleState} placeholder='bookTtile' onChange={(e) => setBookTitleState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookID' type='text' size='40' value={bookIDState} placeholder='bookID' onChange={(e) => setBookIDState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookISBN' type='text' size='40' value={bookISBNState} placeholder='bookISBN' onChange={(e) => setBookISBNState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookAuthorURL' type='text' size='40' value={bookAuthorURLState} placeholder='bookAuthorURL' onChange={(e) => setBookAuthorURLState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookAuthor' type='text' size='40' value={bookAuthorState} placeholder='bookAuthor' onChange={(e) => setBookAuthorState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookRating' type='text' size='40' value={bookRatingState} placeholder='bookRating' onChange={(e) => setBookRatingState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookRatingCount' type='text' size='40' value={bookRatingCountState} placeholder='bookRatingCount' onChange={(e) => setBookRatingCountState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookReviewCount' type='text' size='40' value={bookReviewCountState} placeholder='bookReviewCount' onChange={(e) => setBookReviewCountState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookImageURL' type='text' size='40' value={bookImageURLState} placeholder='bookImageURL' onChange={(e) => setBookImageURLState(e.target.value)}/>
</div>
<div style={center}>
<input id='bookSimilarBooks' type='text' size='40' value={bookSimilarBooksState} placeholder='bookSimilarBooks' onChange={(e) => setBookSimilarBooksState(e.target.value)}/>
</div>
</form>
}
{formState === 'showAuthor' &&
<form style={{border:'2px solid silver', marginLeft:'180px', marginRight: '180px'}}>
<div style={center}>
<input id='authorName' style={center} type='text' size='40' value={authorNameState} placeholder='authorName' onChange={(e) => setAuthorNameState(e.target.value)}/>
</div>
<div style={center}>
<input id='authorURL' style={center} type='text' size='40' value={authorURLState} placeholder='authorURL' onChange={(e) => setAuthorURLState(e.target.value)}/>
</div>
<div style={center}>
<input id='authorID' type='text' size='40' value={authorIDState} placeholder='authorID' onChange={(e) => setAuthorIDState(e.target.value)}/>
</div>
<div style={center}>
<input id='authorRating' type='text' size='40' value={authorRatingState} placeholder='authorRating' onChange={(e) => setAuthorRatingState(e.target.value)}/>
</div>
<div style={center}>
<input id='authorRatingCount' type='text' size='40' value={authorRatingCountState} placeholder='authorRatingCount' onChange={(e) => setAuthorRatingCountState(e.target.value)}/>
</div>
<div style={center}>
<input id='authorReviewCount' type='text' size='40' value={authorReviewCountState} placeholder='authorReviewCount' onChange={(e) => setAuthorReviewCountState(e.target.value)}/>
</div>
<div style={center}>
<input id='authorImageURL' type='text' size='40' value={authorImageURLState} placeholder='authorImageURL' onChange={(e) => setAuthorImageURLState(e.target.value)}/>
</div>
<div style={center}>
<input id='authorRelatedAuthors' type='text' size='40' value={authorRelatedAuthorsState} placeholder='authorRelatedAuthors' onChange={(e) => setAuthorRelatedAuthorsState(e.target.value)}/>
</div>
<div style={center}>
<input id='authorBooks' type='text' size='40' value={authorAuthorBooksState} placeholder='authorBooks' onChange={(e) => setAuthorAuthorBooksState(e.target.value)}/>
</div>
</form>
}
{formState !== 'hide' &&
<button style={{marginLeft:600, width:60}} onClick={hideForm}> back </button>
}
<div style={{marginTop:20, display: 'flex', justifyContent: 'center', alignItems: 'center'}}>
<FourButtons
backColor='blue'
......@@ -157,18 +335,18 @@ function App() {
borderColor='seagreen'
leftMargin={20}
text='POST'
func={showDialog} />
func={post} />
<FourButtons
backColor='red'
borderColor='indianred'
leftMargin={20}
text='DELETE'
func={mesg} />
</div>
<div style={{marginTop:20, display: 'flex', justifyContent: 'center', alignItems: 'center'}}>
<textarea id="board" name="returnData" rows="6" cols="50" value={text} readOnly={true}>
</textarea>
func={delete_data} />
</div>
{renderState === true &&
<div>
<ElementDisplay data={dataState}/>
</div>}
</div>
);
}
......
web/app/src/Components/AuthorDisplay.js deleted 100644 → 0
+ 0
− 16
View file @ bc96133c
const AuthorDisplay = (authorFile) => {
return (
<table border="1" style="width:100%;">
<tr>
<td>Cell 1</td>
<td>Cell 2</td>
</tr>
<tr>
<td>Cell 3</td>
<td>Cell 4</td>
</tr>
</table>
)
}
export default AuthorDisplay
web/app/src/Components/BookDisplay.js deleted 100644 → 0
+ 0
− 9
View file @ bc96133c
const Display = (bookFile) => {
return (
<div>
</div>
)
}
export default Display
web/app/src/Components/ElementDisplay.js 0 → 100644
+ 24
− 0
View file @ ac4bbd40
import MUIDataTable from "mui-datatables";
const ElementDisplay = (props) => {
var fields = Object.keys(props.data)
var values = Object.values(props.data)
console.log(Object.values(props.data))
console.log(Object.keys(props.data))
const columns = fields
const data = [values]
const options = {
'responsive':'vertical'
}
return (
<MUIDataTable
title={"GET Data"}
data={data}
columns={columns}
options={options}
/>
)
}
export default ElementDisplay
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment