# Copyright (c) 2013, 2017, Combine Control Systems AB
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the Combine Control Systems AB nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.
# IN NO EVENT SHALL COMBINE CONTROL SYSTEMS AB BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
In Sympathy for Data, the action of pointing out where data is located and
actual import of data are separated into two different categories of
nodes. The internal data type Datasource is used to carry the information
about the location of the data to the import nodes.
There exist two nodes for establishing paths to locations with data, either you
are interested in a single source of data, :ref:`Datasource`, or several
sources, :ref:`org.sysess.sympathy.datasources.filedatasourcemultiple`. The
single source can either be a data file or a location in a data base. While for
multiple sources only several data files are handled.
"""
import os
from collections import OrderedDict
import fnmatch
from sympathy.api import node as synode
from sympathy.api import datasource as dsrc
from sympathy.api.nodeconfig import Port, Ports, Tag, Tags
from sympathy.api import qt2 as qt_compat
QtCore = qt_compat.QtCore
QtWidgets = qt_compat.import_module('QtWidgets')
MAX_DISPLAYED_FILES = 500
TIME_EXCEEDED_MSG = (
f'Preview looked in {MAX_DISPLAYED_FILES} directories '
'(and then stopped searching to avoid excessive wait).\n'
'\nPlease execute the node to get all results.')
class SuperNode(object):
author = "Alexander Busck"
version = '1.1'
icon = 'datasource.svg'
tags = Tags(Tag.Input.Import)
class FileDatasourceWidget(QtWidgets.QWidget):
def __init__(self, ctx, parameters, parent=None):
super(FileDatasourceWidget, self).__init__(parent)
self._parameters = parameters
self._init_gui()
self._init_gui_from_parameters()
def _init_gui(self):
vlayout = QtWidgets.QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
file_widget = QtWidgets.QWidget()
file_vlayout = QtWidgets.QVBoxLayout()
file_widget.setLayout(file_vlayout)
db_widget = QtWidgets.QWidget()
db_vlayout = QtWidgets.QVBoxLayout()
odbc_vlayout = QtWidgets.QVBoxLayout()
sqlalchemy_vlayout = QtWidgets.QVBoxLayout()
db_widget.setLayout(db_vlayout)
url_widget = QtWidgets.QWidget()
url_vlayout = QtWidgets.QVBoxLayout()
url_widget.setLayout(url_vlayout)
self._filename_widget = self._parameters['filename'].gui()
self._db_driver = self._parameters['db_driver']
self._db_servername = self._parameters['db_servername']
self._db_databasename = self._parameters['db_databasename']
self._db_user = self._parameters['db_user']
self._db_password = self._parameters['db_password']
self._db_driver_widget = self._db_driver.gui()
self._db_servername_widget = self._db_servername.gui()
self._db_databasename_widget = self._db_databasename.gui()
self._db_user_widget = self._db_user.gui()
self._db_password_widget = self._db_password.gui()
self._connection_widget = (
self._parameters['db_connection_string'].gui())
self._db_sqlalchemy_engine_url = (
self._parameters['db_sqlalchemy_engine_url'].gui())
self._db_method = (
self._parameters['db_method'].gui())
self._type_selector = QtWidgets.QComboBox()
self._url_str_widget = self._parameters['url_str'].gui()
self._url_env_widget = self._parameters['url_env'].gui()
self._type_selector.addItem('File')
self._type_selector.addItem('Database')
self._type_selector.addItem('Url')
self._type_selector.setCurrentIndex(
self._parameters['datasource_type'].value[0])
self._datasource_stackwidget = QtWidgets.QStackedWidget()
file_vlayout.addWidget(self._filename_widget)
url_vlayout.addWidget(self._url_str_widget)
url_vlayout.addWidget(self._url_env_widget)
self._odbc_split_widgets = [
self._db_driver_widget,
self._db_servername_widget,
self._db_databasename_widget,
self._db_user_widget,
self._db_password_widget]
self._connection_widget_active = False
file_vlayout.addStretch()
url_vlayout.addStretch()
self._sqlalchemy_group = QtWidgets.QGroupBox('SQLAlchemy settings')
self._sqlalchemy_group.setLayout(sqlalchemy_vlayout)
self._odbc_group = QtWidgets.QGroupBox('ODBC settings')
self._odbc_group.setLayout(odbc_vlayout)
self._db_methods = OrderedDict(
zip(_db_options, [self._odbc_group, self._sqlalchemy_group]))
for odbc_split_widget in self._odbc_split_widgets:
odbc_vlayout.addWidget(odbc_split_widget)
for odbc_split in [self._db_driver,
self._db_servername,
self._db_databasename,
self._db_user,
self._db_password]:
odbc_split.value_changed.add_handler(self._odbc_split_changed)
odbc_vlayout.addWidget(self._connection_widget)
sqlalchemy_vlayout.addWidget(self._db_sqlalchemy_engine_url)
db_vlayout.addWidget(self._db_method)
db_vlayout.addWidget(self._sqlalchemy_group)
db_vlayout.addWidget(self._odbc_group)
self._datasource_stackwidget.addWidget(file_widget)
self._datasource_stackwidget.addWidget(db_widget)
self._datasource_stackwidget.addWidget(url_widget)
vlayout.addWidget(self._type_selector)
vlayout.addWidget(self._datasource_stackwidget)
self.setLayout(vlayout)
self._type_selector.currentIndexChanged[int].connect(
self._type_changed)
self._db_method_changed(self._parameters['db_method'].value)
self._db_method.valueChanged.connect(
self._db_method_changed)
def _init_gui_from_parameters(self):
try:
index = self._parameters['datasource_type'].value[0]
except KeyError:
index = 0
self._datasource_stackwidget.setCurrentIndex(index)
def _type_changed(self, index):
self._parameters['datasource_type'].value = [index]
self._datasource_stackwidget.setCurrentIndex(index)
def _odbc_split_changed(self):
args = [
self._db_driver.selected,
self._db_servername.value,
self._db_databasename.value,
self._db_user.value,
self._db_password.value]
tabledata = dsrc.File.to_database_dict(*args, db_method='ODBC')
self._connection_widget.set_value(tabledata['path'])
def _db_method_changed(self, value):
for key, db_method in self._db_methods.items():
db_method.setEnabled(key == value)
def datasource_factory(datasource, parameters):
try:
datasource_type = parameters['datasource_type'].selected
except KeyError:
datasource_type = 'File'
if datasource_type == 'File':
tabledata = datasource.to_file_dict(
os.path.abspath(parameters['filename'].value))
elif datasource_type == 'Database':
db_method = parameters['db_method'].value
if db_method == 'ODBC':
args = [
parameters['db_driver'].selected,
parameters['db_servername'].value,
parameters['db_databasename'].value,
parameters['db_user'].value,
parameters['db_password'].value,
parameters['db_connection_string'].value]
elif db_method == 'SQLAlchemy':
args = [
parameters['db_sqlalchemy_engine_url'].value]
tabledata = datasource.to_database_dict(*args, db_method=db_method)
elif datasource_type == 'Url':
url_env = parameters['url_env'].value
url_env = OrderedDict(url_env or [])
tabledata = datasource.to_url_dict(
url=parameters['url_str'].value,
env=url_env)
else:
assert(False)
return tabledata
_db_options = ['ODBC', 'SQLAlchemy']
_type_options = ['File', 'Database', 'Url']
[docs]class FileDatasource(SuperNode, synode.Node):
"""
Create Datasource with path to a data source.
A datasource represents a way of obtaining data and supports three
different formats: File (local file), Database (configurable to use
ODBC or sqlalchemy) and URL (for example, http request).
"""
name = 'Datasource'
nodeid = 'org.sysess.sympathy.datasources.filedatasource'
related = ['org.sysess.sympathy.datasources.filedatasourcemultiple']
outputs = Ports([Port.Datasource(
'Datasource with path to file', name='port1', scheme='text')])
parameters = synode.parameters()
parameters.set_string(
'filename', label='Filename',
description='A filename including path if needed',
editor=synode.Util.filename_editor(['Any files (*)']))
parameters.set_string(
'db_sqlalchemy_engine_url', label='SQLAlchemy engine URL',
value='mssql+pyodbc:///',
description='SQLAlchemy engine URL for connecting to the database')
parameters.set_string(
'db_method',
label='Database connection method',
editor=synode.Util.combo_editor(options=_db_options),
value=_db_options[0],
description=(
'Select which Database connection method that you want to use.'))
parameters.set_list(
'db_driver', ['SQL Server'], label='Database driver',
description='Database driver to use.',
editor=synode.Util.combo_editor())
parameters.set_string(
'db_servername', label='Server name',
description='A valid name to a database server.')
parameters.set_string(
'db_databasename', label='Database name',
description='The name of the database.')
parameters.set_string(
'db_user', label='User',
description='A valid database user.')
parameters.set_string(
'db_password', label='Password',
description='A valid password for the selected user.')
parameters.set_string(
'db_connection_string', label='Connection string',
description='A connection string that will override other settings.')
parameters.set_string(
'url_str', label='URL string',
description='Uniform Resource Locator (URL) string',
value='')
parameters.set_json(
'url_env', label='Url environment',
description='Environment for URL. '
'Used as headers in case of HTTP URL. '
'Input as a Python expression that evaluates to.',
value=[],
editor=synode.editors.table_editor(
headers=['Name', 'Value'],
types=['text', 'text'],
unique=['Name']))
parameters.set_list(
'datasource_type', _type_options,
label='Datasource type',
description='Type of datasource.')
INTERACTIVE_NODE_ARGUMENTS = {
'uri': ['filename', 'value']
}
def update_parameters(self, old_parameters):
datasource_type = old_parameters['datasource_type']
if 'Url' not in datasource_type.list:
datasource_type.list = _type_options
def exec_parameter_view(self, ctx):
return FileDatasourceWidget(
ctx, ctx.parameters)
def execute(self, ctx):
"""Execute"""
ctx.output['port1'].encode(
datasource_factory(ctx.output['port1'],
ctx.parameters))
def find_filenames(directory, pattern, fully_qualified=True,
recursive=False, include_files=True,
include_dirs=True, max_length=None):
if not directory.strip():
directory = os.path.abspath('.')
def inner():
length = 0
for i, (root, dirnames, filenames) in enumerate(
os.walk(directory)):
if max_length and (i > max_length or length > max_length):
# Signal that we gave up.
yield None
break
inputs = []
outputs = []
if include_dirs:
inputs.extend(dirnames)
if include_files:
inputs.extend(filenames)
for filename in fnmatch.filter(inputs, pattern):
if fully_qualified:
outputs.append(os.path.normpath(
os.path.join(root, filename)))
else:
outputs.append(os.path.normpath(filename))
length += len(outputs)
yield outputs
matches = []
gave_up = False
results = inner()
if not recursive:
try:
matches.extend(next(results))
except StopIteration:
pass
else:
for outputs in results:
if outputs is not None:
matches.extend(outputs)
else:
gave_up = True
if max_length:
matches = matches[:max_length]
return gave_up, matches
class FileDatasourcesWidget(QtWidgets.QWidget):
def __init__(self, ctx, parameter_root, parent=None):
super(FileDatasourcesWidget, self).__init__(parent)
self._parameter_root = parameter_root
self._init_gui()
def _init_gui(self):
vlayout = QtWidgets.QVBoxLayout()
vlayout.setContentsMargins(0, 0, 0, 0)
use_recursive_widget = self._parameter_root['recursive'].gui()
self._directory_widget = self._parameter_root['directory'].gui()
search_pattern_widget = self._parameter_root['search_pattern'].gui()
include_types_widget = self._parameter_root['include_types'].gui()
self._file_widget = QtWidgets.QListWidget()
self._file_widget.setAlternatingRowColors(True)
self._file_widget.setSelectionMode(
QtWidgets.QAbstractItemView.NoSelection)
_file_widget_label = QtWidgets.QLabel(
f'Preview ({MAX_DISPLAYED_FILES} first hits displayed)')
vlayout.addWidget(use_recursive_widget)
vlayout.addWidget(self._directory_widget)
vlayout.addWidget(search_pattern_widget)
vlayout.addWidget(include_types_widget)
vlayout.addWidget(_file_widget_label)
vlayout.addWidget(self._file_widget)
self.setLayout(vlayout)
self._update_filewidget()
use_recursive_widget.stateChanged.connect(
self._update_filewidget)
self._directory_widget.editor().dialogChanged[str].connect(
self._update_filewidget)
self._directory_widget.editor().text_changed.connect(
self._update_filewidget)
search_pattern_widget.valueChanged.connect(
self._update_filewidget)
include_types_widget.editor().currentIndexChanged.connect(
self._update_filewidget)
def _update_filewidget(self, *args):
gave_up, selected_fq_filenames = find_filenames(
self._directory_widget.editor().filename(),
self._parameter_root['search_pattern'].value,
fully_qualified=True,
max_length=MAX_DISPLAYED_FILES,
recursive=self._parameter_root['recursive'].value,
**_included_types(self._parameter_root['include_types'].value))
preview_list = selected_fq_filenames
if gave_up:
preview_list = preview_list + [TIME_EXCEEDED_MSG]
self._file_widget.clear()
self._file_widget.addItems(preview_list)
_type_choice_files, _type_choice_dirs, _type_choice_both = _type_choices = [
'Files', 'Directories', 'Files and Directories']
def _add_include_types_param(parameters):
parameters.set_string(
'include_types', label='Include',
description='Choose whether matching files, directories or both '
'should be included in the result',
editor=synode.editors.combo_editor(options=_type_choices),
value=_type_choice_files)
def _included_types(value):
include_files = False
include_dirs = False
if value == _type_choice_files:
include_files = True
elif value == _type_choice_dirs:
include_dirs = True
else:
include_files = True
include_dirs = True
return {'include_files': include_files,
'include_dirs': include_dirs}
[docs]class FileDatasourceMultiple(SuperNode, synode.Node):
"""
Create Datasources with paths to data sources.
"""
name = 'File Datasources'
description = 'Select data sources.'
nodeid = 'org.sysess.sympathy.datasources.filedatasourcemultiple'
related = ['org.sysess.sympathy.datasources.filedatasource']
outputs = Ports([Port.Datasources(
'Datasources with paths files',
name='port1', scheme='text')])
parameters = synode.parameters()
parameters.set_boolean(
'recursive', value=False, label='Recursive',
description=('If unchecked, only the selected directory will be '
'searched. If checked, all subdirectories of the '
'selected directory will also be searched.'))
parameters.set_string(
'directory', label='Directory',
description=('Directory where to search for files.'),
editor=synode.Util.directory_editor())
parameters.set_string(
'search_pattern', value='*', label='Search pattern',
description='A wildcard pattern which the filenames must match.')
_add_include_types_param(parameters)
def update_parameters(self, parameters):
if ('include_types' not in parameters and
not parameters['recursive'].value):
_add_include_types_param(parameters)
parameters['include_types'].value = _type_choice_both
def exec_parameter_view(self, ctx):
return FileDatasourcesWidget(ctx, ctx.parameters)
def execute(self, ctx):
"""Create a list of datasources and add them to the output
file.
"""
selected_fq_filenames = find_filenames(
ctx.parameters['directory'].value,
ctx.parameters['search_pattern'].value,
fully_qualified=True,
recursive=ctx.parameters['recursive'].value,
**_included_types(
ctx.parameters['include_types'].value))[1]
for fq_filename in selected_fq_filenames:
datasource = dsrc.File()
datasource.encode_path(os.path.abspath(fq_filename))
ctx.output['port1'].append(datasource)
