515 lines
24 KiB
Python
515 lines
24 KiB
Python
# Part of Odoo. See LICENSE file for full copyright and licensing details.
|
|
import base64
|
|
import html
|
|
import io
|
|
import json
|
|
import logging
|
|
from psycopg2.errors import SerializationFailure
|
|
import re
|
|
import requests
|
|
from lxml import etree
|
|
from lxml import html as lxml_html
|
|
import tarfile
|
|
import traceback
|
|
from urllib.parse import urlparse
|
|
|
|
from odoo import api, fields, models
|
|
from odoo.addons.iap.tools.iap_tools import iap_jsonrpc
|
|
from odoo.exceptions import AccessError
|
|
from odoo.tools import LazyTranslate
|
|
from odoo.tools.urls import urljoin as url_join
|
|
|
|
_lt = LazyTranslate(__name__)
|
|
|
|
DEFAULT_WSS_ENDPOINT = 'https://iap-scraper.odoo.com/'
|
|
GET_RESULT_TIMEOUT_SECONDS = 3600 # 1 hour
|
|
STATUS_MESSAGES = {
|
|
'success': _lt("Success"),
|
|
'processing': _lt("Processing"),
|
|
'waiting': _lt("Waiting for the server to process the request"),
|
|
'done': _lt("Done, website generated"),
|
|
'error_maintenance': _lt("Server is currently under maintenance. Please retry later"),
|
|
'error_internal': _lt("An error occurred"),
|
|
'error_invalid_url': _lt("Invalid url"),
|
|
'error_banned_url': _lt("Banned url"),
|
|
'error_invalid_dbuuid': _lt("Invalid dbuuid"),
|
|
'error_too_many_pages': _lt("The request asks for too many pages"),
|
|
'error_unsupported_version': _lt("Version is unsupported"),
|
|
'error_invalid_token': _lt("Invalid token"),
|
|
'error_concurrent_request': _lt("Number of concurrent requests exceeded"),
|
|
'error_allowed_request_exhausted': _lt("Number of allowed requests exhausted"),
|
|
'error_invalid_import_products': _lt("Invalid import products"),
|
|
'error_invalid_request_uuid': _lt("Could not fetch result, invalid output uuid or result expired"),
|
|
'error_request_still_processing': _lt("Request is still processing, result not available yet"),
|
|
'error_attachment_not_found': _lt("Attachment not found"),
|
|
'errror_website_not_supported': _lt("Website not supported"),
|
|
'error_website_blocked': _lt("Website blocked or unreachable"),
|
|
'error_navigation_timeout': _lt("The requested url took too long to load and is probably unreachable from the server"),
|
|
'error_url_redirection': _lt("The requested url redirected to another url, try again with the redirected url"),
|
|
'error_duplicated_request': _lt("The same request has just been made, please wait before sending another identical request"),
|
|
}
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class Website_GeneratorRequest(models.Model):
|
|
_name = 'website_generator.request'
|
|
_description = "Website Generator Request"
|
|
|
|
target_url = fields.Char(string="URL to scrape", required=True)
|
|
additional_urls = fields.Char(string="Additional URLs")
|
|
page_count = fields.Integer(string="Number of pages")
|
|
uuid = fields.Char(string="Output UUID generated from Website Scraper Server")
|
|
status = fields.Char(string="Status", default='waiting')
|
|
status_message = fields.Char(string="Status Message", compute='_compute_status_message')
|
|
version = fields.Char(string="Version", default='2.2.0')
|
|
website_id = fields.Many2one('website', string="Website", ondelete='cascade')
|
|
notified = fields.Boolean(string="Notified", default=False)
|
|
|
|
@api.model_create_multi
|
|
def create(self, vals_list):
|
|
wg_requests = super().create(vals_list)
|
|
for req in wg_requests:
|
|
# If there is already a uuid, it means the request was already
|
|
# created via the odoo.com start trial.
|
|
if not req.uuid:
|
|
ws_endpoint = self.env['ir.config_parameter'].sudo().get_param('website_scraper_endpoint', DEFAULT_WSS_ENDPOINT)
|
|
url = url_join(ws_endpoint, f'/website_scraper/{req.version}/scrape')
|
|
response = iap_jsonrpc(url, params=req._get_call_params())
|
|
if response.get('status') == 'accepted':
|
|
req.uuid = response['uuid']
|
|
req.status = 'waiting'
|
|
else:
|
|
req.status = response.get('status', 'error_internal')
|
|
logger.warning("Error calling WS server: %s", req.status)
|
|
|
|
self.env.ref("website_generator.cron_get_result").toggle(model=self._name, domain=[])
|
|
return wg_requests
|
|
|
|
@api.depends('status')
|
|
def _compute_status_message(self):
|
|
for record in self:
|
|
record.status_message = self.env._(STATUS_MESSAGES.get(record.status, STATUS_MESSAGES['error_internal'])) # pylint: disable=gettext-variable
|
|
|
|
def _get_call_params(self):
|
|
ICP = self.env['ir.config_parameter'].sudo()
|
|
params = {
|
|
'url': self.target_url,
|
|
'additional_urls': self.additional_urls,
|
|
'token': ICP.get_param('website_generator.token', None),
|
|
'dbuuid': ICP.get_param('database.uuid'),
|
|
}
|
|
if self.page_count:
|
|
params['page_count'] = self.page_count
|
|
return params
|
|
|
|
@api.model
|
|
def get_result_waiting_requests(self):
|
|
""" This method is called by the CRON job which is toggled by the creation of a request. """
|
|
ready_requests = self.search([
|
|
('status', 'in', ['waiting', 'error_request_still_processing', 'error_maintenance']),
|
|
])
|
|
for request in ready_requests:
|
|
request._call_server_get_result()
|
|
|
|
from_cron = bool(self.env.context.get('cron_id'))
|
|
if from_cron:
|
|
pending_requests = ready_requests.filtered(lambda req: req.status in ['waiting', 'error_request_still_processing', 'error_maintenance'])
|
|
if not pending_requests:
|
|
self.env['ir.cron']._commit_progress(processed=len(ready_requests), remaining=0, deactivate=True)
|
|
logger.info("Website Generator: No more pending request, disabling 'cron_get_result' cron")
|
|
|
|
def _call_server_get_result(self):
|
|
# Don't inline this method in `get_result_waiting_requests()`, it's
|
|
# needed for ease of development (overridden in custom dev module)
|
|
logger.info("Website Generator: Getting result for request uuid: %s", self.uuid)
|
|
ICP = self.env['ir.config_parameter'].sudo()
|
|
data = {
|
|
'uuid': self.uuid,
|
|
'dbuuid': ICP.get_param('database.uuid'),
|
|
# This is called by a CRON, we are not in a website context,
|
|
# this will always shortcut to ICP, so we can at least have a domain to go back to.
|
|
'db_url': self.get_base_url(),
|
|
}
|
|
ws_endpoint = ICP.get_param('website_scraper_endpoint', DEFAULT_WSS_ENDPOINT)
|
|
url = url_join(ws_endpoint, f'/website_scraper/{self.version}/get_result')
|
|
response = requests.get(url, params=data, timeout=GET_RESULT_TIMEOUT_SECONDS)
|
|
|
|
# /get_result is not protected by token
|
|
data['token'] = ICP.get_param('website_generator.token', None)
|
|
|
|
# Check the response
|
|
try:
|
|
if response.status_code != 200:
|
|
self.status = 'error_internal'
|
|
logger.warning("Error calling WS server: Status code %s", response.status_code)
|
|
return
|
|
# TODO: Find a better way to check if the response failed or not
|
|
elif response.headers.get('Content-Type', '') != 'application/x-tar':
|
|
# An error occurred, getting the status from the response
|
|
# On top of real errors, some "normal" status could be returned
|
|
# here as `error_request_still_processing` if the scraping is
|
|
# not yet finished.
|
|
self.status = response.json().get('status', 'error_internal')
|
|
return
|
|
except Exception:
|
|
# If the response is not JSON, it means the request was successful
|
|
pass
|
|
|
|
try:
|
|
tar_gz_file = io.BytesIO(response.content)
|
|
with tarfile.open(fileobj=tar_gz_file, mode='r:gz') as tar:
|
|
website, _ = self._generate_site(tar)
|
|
self.status = 'done'
|
|
# Send email to the client.
|
|
website = self.env['website'].get_current_website()
|
|
mail_template = self.env.ref('website_generator.email_template_website_scrapped')
|
|
email_values = {'email_to': self.env.company.email_formatted, 'website_url': website.get_base_url()}
|
|
mail_template.with_context(email_values).send_mail(
|
|
website.id,
|
|
force_send=True,
|
|
email_values=None,
|
|
)
|
|
|
|
# Report OK to IAP (success)
|
|
logger.info("Website Generator: Reporting OK for request uuid: %s", self.uuid)
|
|
url = url_join(ws_endpoint, f'/website_scraper/{self.version}/report_ok')
|
|
self._report_to_iap(url, data)
|
|
except SerializationFailure:
|
|
raise
|
|
except Exception as e:
|
|
# Defensive programming: if necessary info is missing, stop and warn IAP
|
|
# (should not happen, but just in case of a future changes in the WS server)
|
|
# Rollback the transaction to avoid the creation of the website
|
|
self.env.cr.rollback()
|
|
self.status = 'error_internal'
|
|
logger.exception("Error building the website: %s", e)
|
|
data['error'] = traceback.format_exc()
|
|
|
|
# Report KO to IAP (useful for spotting critical errors)
|
|
logger.info("Website Generator: Reporting KO for request uuid: %s", self.uuid)
|
|
url = url_join(ws_endpoint, f'/website_scraper/{self.version}/report_ko')
|
|
self._report_to_iap(url, data)
|
|
|
|
def _report_to_iap(self, url, data):
|
|
try:
|
|
resp = iap_jsonrpc(url, params=data)
|
|
if resp.get('status') != 'ok':
|
|
logger.warning("Error reporting to WS server: %s", resp.get('status'))
|
|
except AccessError as e:
|
|
logger.warning("Error reporting to WS server: %s", e)
|
|
|
|
def _generate_site(self, tar):
|
|
odoo_blocks = self._load_input(tar)
|
|
odoo_blocks['direct_html_replacements_mapping'] = {}
|
|
website = self._get_website(odoo_blocks, tar)
|
|
# Generate the images attachments (Modifies odoo_blocks in place)
|
|
self._save_images_as_attachments(odoo_blocks, tar)
|
|
self._create_model_records(tar, odoo_blocks)
|
|
# Create redirects, modifies odoo_blocks in place
|
|
self._apply_all_html_replacements(odoo_blocks)
|
|
self._generate_pages(website, odoo_blocks)
|
|
return website, odoo_blocks
|
|
|
|
def _apply_all_html_replacements(self, odoo_blocks):
|
|
direct_html_replacements_mapping = odoo_blocks['direct_html_replacements_mapping']
|
|
sorted_original_html = sorted(direct_html_replacements_mapping.keys(), key=len, reverse=True)
|
|
# Use a regular expression to match any of the replacements
|
|
pattern_sorted_html = r'(' + '|'.join(map(re.escape, sorted_original_html)) + r')'
|
|
|
|
homepage = odoo_blocks['homepage']
|
|
image_replacement_mapping = homepage['image_replacement_mapping']
|
|
homepage['body_html'] = self._apply_html_replacements(homepage.get('body_html', []), pattern_sorted_html, direct_html_replacements_mapping, image_replacement_mapping)
|
|
|
|
footer = homepage.get('footer', [])
|
|
if footer:
|
|
homepage['footer'] = self._apply_html_replacements(footer, pattern_sorted_html, direct_html_replacements_mapping, image_replacement_mapping)
|
|
|
|
header_buttons = homepage.get('header', {}).get('buttons', [])
|
|
for button in header_buttons:
|
|
if button.get('href') in direct_html_replacements_mapping:
|
|
button['href'] = direct_html_replacements_mapping[button['href']]
|
|
|
|
# Update the html urls for all pages
|
|
for page_name, page_dict in odoo_blocks.get('pages', {}).items():
|
|
image_replacement_mapping = page_dict['image_replacement_mapping']
|
|
odoo_blocks['pages'][page_name]['body_html'] = self._apply_html_replacements(page_dict.get('body_html', []), pattern_sorted_html, direct_html_replacements_mapping, image_replacement_mapping)
|
|
|
|
def _create_model_records(self, tar, odoo_blocks):
|
|
# Each override will call super and create it's model records as well as any redirects it needs.
|
|
pass
|
|
|
|
def _load_input(self, tar):
|
|
# Don't inline this method in `_generate_site()`, it's needed for ease
|
|
# of development (overridden in custom dev module)
|
|
return json.load(tar.extractfile('out.json'))
|
|
|
|
def _generate_pages(self, website, odoo_blocks):
|
|
# Create pages
|
|
for page_url, page_data in odoo_blocks.get('pages', {}).items():
|
|
if 'body_html' in page_data:
|
|
# Create page
|
|
new_page_info = website.with_context(website_id=website.id).new_page(page_data['name'])
|
|
new_page = self.env['website.page'].browse(new_page_info['page_id'])
|
|
# force url to the one provided, don't use the slugified one
|
|
new_page.url = page_url
|
|
new_page.is_published = True
|
|
# Create page content
|
|
new_page._construct_page(page_data)
|
|
|
|
# Remove the default homepage
|
|
homepage_url = website.homepage_url or '/'
|
|
self.env['website.page'].search([('website_id', '=', website.id), ('url', '=', homepage_url)]).unlink()
|
|
homepage_info = website.with_context(website_id=website.id).new_page('Home')
|
|
homepage = self.env['website.page'].browse(homepage_info['page_id'])
|
|
homepage.write({
|
|
'url': '/',
|
|
'is_published': True,
|
|
})
|
|
# Create home page content
|
|
homepage._construct_homepage(odoo_blocks['homepage'])
|
|
|
|
def _get_website(self, odoo_blocks, tar):
|
|
website = self.env['website'].get_current_website()
|
|
self.write({'website_id': website.id})
|
|
website_info = odoo_blocks.get('website')
|
|
if not website_info:
|
|
raise ValueError("Website info not found in the input")
|
|
homepage_url = odoo_blocks.get('homepage', {}).get('url')
|
|
if not homepage_url:
|
|
raise ValueError("Homepage url not found in the input")
|
|
website_name = urlparse(homepage_url).netloc.removeprefix('www.')
|
|
website_values = {'name': website_name, **website_info.get('social_media_links', {})}
|
|
|
|
# Add logo
|
|
logo_filename = website_info.get('logo')
|
|
if logo_filename:
|
|
image = self._get_image_data(tar, logo_filename)
|
|
if image:
|
|
website_values['logo'] = base64.b64encode(image).decode()
|
|
|
|
website.update(website_values)
|
|
return website
|
|
|
|
def _save_images_as_attachments(self, odoo_blocks, tar):
|
|
all_images = odoo_blocks['website'].get('all_images', {})
|
|
# Create attachments for all images (uncropped)
|
|
attachments_url_src = {}
|
|
for img_url, img_name in all_images.items():
|
|
attachment = self.try_create_image_attachment(img_name, img_url, tar)
|
|
if attachment:
|
|
attachments_url_src[img_url] = attachment
|
|
odoo_blocks['direct_html_replacements_mapping'].update({html.escape(k): v.image_src for k, v in attachments_url_src.items()})
|
|
|
|
# Create attachments for all images (cropped)
|
|
for page_dict in [odoo_blocks['homepage']] + list(odoo_blocks.get('pages', {}).values()):
|
|
customized_images = page_dict.get('images_to_customize', [])
|
|
page_dict['image_replacement_mapping'] = {}
|
|
for ws_id, image_customizations in customized_images.items():
|
|
# Note, we give the 'ws_id' as the image_url because we may have multiple images
|
|
# with the same url but cropped differently (where the image_url is the
|
|
# downloaded image url from the original website).
|
|
img_name = image_customizations.get('filename')
|
|
mimetype = image_customizations.get('mimetype')
|
|
url = image_customizations.get('url')
|
|
if not img_name or not mimetype or not url or not ws_id:
|
|
continue
|
|
|
|
# Link to attachment
|
|
attachment = attachments_url_src.get(url)
|
|
if not attachment:
|
|
continue
|
|
|
|
# Create a new attachment if needed
|
|
src = attachment.image_src
|
|
if image_customizations.get('create_new_attachment', False):
|
|
new_attachment = self.try_create_image_attachment(img_name, ws_id, tar)
|
|
if new_attachment:
|
|
src = new_attachment.image_src
|
|
|
|
attributes = {
|
|
'src': src,
|
|
'data-attachment-id': attachment.id,
|
|
'data-original-id': attachment.id,
|
|
'data-original-src': attachment.image_src,
|
|
'data-mimetype': mimetype,
|
|
'data-mimetype-before-conversion': mimetype,
|
|
}
|
|
|
|
# Apply the cropping attributes
|
|
cropping_dimensions = image_customizations.get('cropping_coords', {})
|
|
if cropping_dimensions:
|
|
attributes.update({
|
|
'data-x': cropping_dimensions['x'],
|
|
'data-y': cropping_dimensions['y'],
|
|
'data-width': cropping_dimensions['width'],
|
|
'data-height': cropping_dimensions['height'],
|
|
'data-scale-x': 1,
|
|
'data-scale-y': 1,
|
|
'data-aspect-ratio': '0/0',
|
|
})
|
|
|
|
color_filter = image_customizations.get('filter', {})
|
|
if color_filter:
|
|
rgba = f'rgba({int(color_filter["coords"][0] * 255)}, {int(color_filter["coords"][1] * 255)}, {int(color_filter["coords"][2] * 255)}, {color_filter["alpha"]})'
|
|
attributes.update({
|
|
'data-gl-filter': 'custom',
|
|
'data-filter-options': json.dumps({'filterColor': rgba}, separators=(',', ':')),
|
|
})
|
|
|
|
page_dict['image_replacement_mapping'][str(ws_id)] = attributes
|
|
|
|
def try_create_image_attachment(self, img_name, img_url, tar):
|
|
try:
|
|
# Read from tar
|
|
image_data = self._get_image_data(tar, img_name)
|
|
if not image_data:
|
|
return None
|
|
# Create a new attachment
|
|
att = self.env['ir.attachment'].create({
|
|
'name': img_name,
|
|
'raw': image_data,
|
|
'public': True,
|
|
'res_model': 'ir.ui.view',
|
|
'res_id': 0, # shared between website's pages
|
|
})
|
|
if att and att.image_src:
|
|
return att
|
|
except (AttributeError, TypeError, ValueError) as e:
|
|
# Defensive programming: skip the image if it's invalid
|
|
# (image extension not supported, corrupted metadata, etc.)
|
|
logger.warning("Error attaching image %r : %s", img_url, e)
|
|
return None
|
|
|
|
def _apply_html_replacements(self, body_html, pattern_sorted_html, direct_replacement_mapping,
|
|
image_replacement_mapping, template_key_to_filter_xmlid=None):
|
|
template_key_to_filter_xmlid = template_key_to_filter_xmlid or {}
|
|
template_key_to_filter_id = {
|
|
tk: self.env.ref(xmlid).id
|
|
for tk, xmlid in template_key_to_filter_xmlid.items()
|
|
if self.env.ref(xmlid, raise_if_not_found=False)
|
|
}
|
|
|
|
new_block_list = []
|
|
for block_html in body_html:
|
|
page_html = self._replace_in_string(block_html, pattern_sorted_html, direct_replacement_mapping)
|
|
page_html = self._update_html(page_html, image_replacement_mapping, template_key_to_filter_id)
|
|
new_block_list.append(page_html)
|
|
return new_block_list
|
|
|
|
def _find_or_create(self, model, domain, vals):
|
|
record = self.env[model].search(domain, limit=1)
|
|
if not record:
|
|
record = self.env[model].create(vals)
|
|
return record
|
|
|
|
def _get_image_data(self, tar, image_name):
|
|
if not image_name:
|
|
return None
|
|
try:
|
|
image_data = tar.extractfile('images/' + image_name).read()
|
|
return image_data
|
|
except (KeyError, AttributeError) as e:
|
|
logger.warning("Image %s not found : %s", image_name, e)
|
|
return None
|
|
|
|
def _get_image_info(self, tar, images, image_file_mappings):
|
|
all_images_info = []
|
|
for image in images:
|
|
image_filename = image_file_mappings.get(image, '')
|
|
image_data = self._get_image_data(tar, image_filename)
|
|
if image_data:
|
|
all_images_info.append({
|
|
'name': image_filename,
|
|
'raw': image_data,
|
|
'base64': base64.b64encode(image_data).decode(),
|
|
})
|
|
return all_images_info
|
|
|
|
@staticmethod
|
|
def _update_html(page_html, image_replacement_mapping, template_key_to_filter_id):
|
|
if not image_replacement_mapping and not template_key_to_filter_id:
|
|
return page_html
|
|
|
|
try:
|
|
container = lxml_html.fragment_fromstring(page_html.replace('\ufeff', ''), create_parent='div')
|
|
except (etree.ParserError, ValueError) as e:
|
|
logger.warning("Could not parse snippet for image replacement: %s", e)
|
|
return page_html
|
|
|
|
for el in container.xpath('.//*[@data-template-key]'):
|
|
target_filter_id = template_key_to_filter_id.get(el.get('data-template-key'))
|
|
if target_filter_id:
|
|
el.set('data-filter-id', str(target_filter_id))
|
|
|
|
for img in container.xpath('.//img[@data-ws_id]'):
|
|
replacement_attrs = image_replacement_mapping.get(img.get('data-ws_id'))
|
|
if not replacement_attrs:
|
|
continue
|
|
|
|
new_img = lxml_html.Element('img')
|
|
for attribute, value in replacement_attrs.items():
|
|
new_img.set(attribute, str(value))
|
|
|
|
# Preserve the original class/style attributes
|
|
img_class = img.get('class')
|
|
if img_class:
|
|
new_img.set('class', img_class)
|
|
img_style = img.get('style')
|
|
if img_style:
|
|
new_img.set('style', img_style)
|
|
|
|
img.getparent().replace(img, new_img)
|
|
|
|
# Serialize only the fragment content, not the temporary wrapper element
|
|
# created by fragment_fromstring(..., create_parent='div').
|
|
return (container.text or '') + ''.join(
|
|
etree.tostring(child, encoding='unicode')
|
|
for child in container
|
|
)
|
|
|
|
@staticmethod
|
|
def _replace_in_string(string, pattern_sorted_html, replacements):
|
|
if not replacements:
|
|
return string
|
|
|
|
def replace_callback(match):
|
|
# Having this callback function is useful for verifying which URLs were replaced.
|
|
matched_url = match.group(0)
|
|
replacement = replacements.get(matched_url)
|
|
if not replacement:
|
|
replacement = matched_url
|
|
logger.warning("Match found but URL %r not found in attachments", matched_url)
|
|
return replacement
|
|
|
|
# Replace all matches with their corresponding replacement
|
|
replaced_string = re.sub(pattern_sorted_html, replace_callback, string)
|
|
return replaced_string
|
|
|
|
@api.model
|
|
def convert_scraping_request_ICP(self):
|
|
ICP = ws_uuid = self.env['ir.config_parameter'].sudo()
|
|
ws_uuid = ICP.get_param('website_generator.iap_ws_uuid', None)
|
|
ws_target_url = ICP.get_param('website_generator.iap_ws_target_url', None)
|
|
|
|
if not (ws_uuid and ws_target_url):
|
|
# TODO: return website configurator?
|
|
return {
|
|
'type': 'ir.actions.act_url',
|
|
'url': '/odoo',
|
|
'target': 'self',
|
|
}
|
|
|
|
self.env['website_generator.request'].create({
|
|
'uuid': ws_uuid,
|
|
'target_url': ws_target_url,
|
|
})
|
|
ICP.set_param('website_generator.iap_ws_uuid', None)
|
|
ICP.set_param('website_generator.iap_ws_target_url', None)
|
|
|
|
return {
|
|
'type': 'ir.actions.act_url',
|
|
'url': "/odoo/action-website_generator.website_generator_screen?reload=true",
|
|
'target': 'self',
|
|
}
|