# (SS,14/08/15) Python script to scrape reviews from Trustpilot
# Version of Python must be 3.4 or greater
# (SS,11/09/15) Changed to use data-reviewmid instead of data-reviewid because data-reviewid is now always a 0

import urllib.request
import datetime
from bs4 import BeautifulSoup
import mysql.connector
import sys
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import email.utils


def fetch_page(AURL): 
  req = urllib.request.urlopen(AURL)  
  return req.read()      
      
def get_trustpilot_reviews(AURL):    
    html = fetch_page(AURL)
    
    soup = BeautifulSoup(html, "html.parser")

    # get the average which is calculated by their special formula, i.e. we can't calculate it from the records unless we have the exact formula
    summary = soup.find("div", { "class" : "summary-rating" })  
    # .decode('ascii') prevents TypeError "Can't convert 'bytes' object to str implicitly"
    summary_rating = summary.find("span", {"itemprop":"ratingValue"}).renderContents().decode('ascii')
    print_log("Average rating: " + summary_rating)
    word_rating = summary.find("div", { "class" : "word-rating" }).getText().strip().splitlines()[0]
    print_log("Rating word: " + word_rating)        
    review_count = summary.find("span", {"itemprop":"reviewCount"}).renderContents().decode('ascii')
    print_log("Total reviews: " + review_count)
    
    # Trustpilot average rating, stars and word
    # 9.0 to 10	  5	Excellent
    # 7.0 to 8.9	4	Great
    # 5.0 to 6.9	3	Average
    # 3.0 to 4.0	2 Poor
    # 1.0 to 2.9	1	Bad
    summary_rating = float(summary_rating)
    if summary_rating >= 9.0:    
      summary_rating_1_to_5 = 5
    elif summary_rating >= 7.0:
      summary_rating_1_to_5 = 4
    elif summary_rating >= 5.0:
      summary_rating_1_to_5 = 3
    elif summary_rating >= 3.0:
      summary_rating_1_to_5 = 2
    else:
      summary_rating_1_to_5 = 1
    
    
    data = (review_count.strip(), summary_rating_1_to_5, word_rating.strip(), summary_rating)
    sql = "UPDATE trustpilot_reviews SET ReviewDateTime = NOW(), ReviewerName = %s, ReviewRating = %s, ReviewTitle = %s, ReviewBody = %s WHERE ReviewID = 1"
    cursor.execute(sql, data)            
    
    
    # find all reviews
    reviews = soup.findAll("div", { "class" : "review" })
    
    count = 0
    new_reviews = 0
    for review in reviews:
      count = count + 1

      # look for trustpilot_review_id already in trustpilot_reviews table
	  # (SS,11/9/15) data-reviewid now always zero, so changed to use data-reviewmid instead which is a hex number 24 characters long
	  #	field TrustPilotReviewID changed from INTEGER(11) to VARCHAR(24)
      trustpilot_review_id = review['data-reviewmid'].strip()

      sql = "SELECT COUNT(*) FROM trustpilot_reviews WHERE TrustpilotReviewID = %s"
      cursor.execute(sql, [trustpilot_review_id])
      result = cursor.fetchone()
      number_of_rows = result[0]      
      
      # only add if same review doesn't already exist
      if number_of_rows == 0:
        new_reviews = new_reviews + 1
        reviewer_date_time = review.find("meta", {"itemprop":"dateCreated"})["content"]
        # format is 2015-07-24T20:58:49.385+00:00, needed to remove the +00:00 bit
        reviewer_date_time = reviewer_date_time[:23]
        reviewer_name = review.find("div", { "class" : "user-review-name" }).find("span").renderContents()      
        review_rating = review.find("div", { "class" : "star-rating" }).find("meta", {"itemprop":"ratingValue"})["content"]       
        review_title = review.find("h3", { "class" : "review-title" }).find("a").renderContents()      
        review_body = review.find("div", { "class" : "review-body" }).renderContents()

        data = (trustpilot_review_id, reviewer_date_time, reviewer_name.strip(), review_rating, review_title.strip(), review_body.strip())
        sql = "INSERT INTO trustpilot_reviews (TrustpilotReviewID, ReviewDateTime, ReviewerName, ReviewRating, ReviewTitle, ReviewBody) VALUES (%s, %s, %s, %s, %s, %s)"
        cursor.execute(sql, data)

    print_log("")
    print_log("Total reviews on page: " + str(count))
    print_log("New reviews: " + str(new_reviews))

    
def get_reviews():
    get_trustpilot_reviews('https://uk.trustpilot.com/review/www.expressmusicstore.co.uk')
   # get_trustpilot_reviews('https://uk.trustpilot.com/review/www.expressmusicstore.co.uk?page=2')


def set_alert(AAlert):
    global FAlert # required to modify a global 
    FAlert = AAlert
      
def get_alert():
    return FAlert

def print_log(AText):
    global FPrintLog # required to modify a global
    print(AText)
    FPrintLog = FPrintLog + AText + '\r\n'
      
def send_email_text(ABody, ASubject, AFrom, ATo, ABcc):
    # Create a text/plain message
    msg = MIMEText(ABody)

    # header    
    if get_alert():
        msg['Subject'] = 'ALERT!!! ' + ASubject
    else:
        msg['Subject'] = ASubject
    msg['From'] = AFrom
    msg['To'] = ATo
    msg['Date'] = email.utils.formatdate(localtime=True)
    if get_alert():       
        msg['X-Priority'] = '2'
            
    # send the messsage using localhost
    s = smtplib.SMTP('localhost')
    s.sendmail(AFrom, [ATo, ABcc], msg.as_string())
    s.quit()     

FAlert = False  # global used to set alert
FPrintLog = ""  # global used to hold log that is also the email body 
        
start = datetime.datetime.now()
print_log('Started: ' + start.strftime('%Y-%m-%d %H:%M:%S'))
print_log('')

try:
  # open connection
  cnx = None
  cursor = None
  cnx = mysql.connector.connect(user='expressmusic', password='bsharp',
                                host='localhost',
                                database='expressmusic')
  cursor = cnx.cursor()

  get_reviews()
except:
  set_alert(True)
  print_log("Error:")
  print_log(str(sys.exc_info()))
  #raise

finally:
  # close connections if they were opened
  if cursor:
    cursor.close()
  if cnx:
    cnx.commit()
    cnx.close()

finish = datetime.datetime.now()
time_taken = finish - start
print_log('')
print_log('Finished: ' + finish.strftime('%Y-%m-%d %H:%M:%S'))
print_log('Time taken: %0.3f seconds' % (time_taken.seconds + time_taken.microseconds / 1E6))

# send the text as an email, from address trustpilot@expressmusicstore.co.uk is not actually a valid account
send_email_text(FPrintLog, 'Express Music Trustpilot Review Scrape Result', 'trustpilot@expressmusicstore.co.uk', 'surinder@itpartnership.com', '')