Dashboard Apps

enrichsdk.app.utils

Utilities meant for the dashboard apps

EnrichAppConfig

Bases: AppConfig

Base class for Dashboard Apps. Each is an instance of Django App.

category = 'default' class-attribute instance-attribute

Category to which the app belongs

instanceid property

Unique id associated with the app

name = 'default' class-attribute instance-attribute

Name of the app

resources property

Attribute to return the resources (a html text string)

get_usecase()

Usecase to which the app belongs

Source code in enrichsdk/app/utils.py

def get_usecase(self):
    """
    Usecase to which the app belongs
    """
    if self.filename is None:
        return {}

    usecase = find_usecase(self.filename)
    self.usecase = copy.copy(usecase)
    return self.usecase

get_default_configuration()

Default description for any app

Source code in enrichsdk/app/utils.py

def get_default_configuration():
    """
    Default description for any app
    """
    return """
    This app requires low code configuration implemented using the app
    sdk or other means. Please get in touch with the platform manager.
    """

get_default_resources()

Default resources for any app

Source code in enrichsdk/app/utils.py

def get_default_resources():
    """
    Default resources for any app
    """
    return """
    """

enrichsdk.app.bases

get_template_dirs()

Template search paths..

Source code in enrichsdk/app/bases/__init__.py

def get_template_dirs():
    """
    Template search paths..
    """
    thisdir = os.path.dirname(os.path.abspath(__file__))
    return [
        os.path.join(thisdir, 'sharedapp', 'templates'),
        os.path.join(thisdir, 'policyapp', 'templates'),
        os.path.join(thisdir, 'singlepageapp', 'templates')
    ]

policyapp

This is the base app setup for applications that have the following component:

policy interface (extensible) api pipeline (backend) result (could be arbitrary)

forms

models

AppPolicyBase

Bases: models.Model

Common policy document

views

AppBase()

A shareable view across apps

Source code in enrichsdk/app/bases/policyapp/views.py

def __init__(self):
    self.name = "outliers"
    self.verbose_name = "Outliers"
    self.category = "outliers"
    self.templates = {
        'policy_index': f'enrichapp/policyapp/policy_index.html',
        'policy_add': f'enrichapp/policyapp/policy_add.html',
        'policy_select_dataset': f'enrichapp/policyapp/policy_select_dataset.html',
        'policy_select_columns': f'enrichapp/policyapp/policy_select_columns.html',
        'policy_select_columns_helper': f'enrichapp/policyapp/policy_select_columns_helper.html',
        'result_index': f'enrichapp/policyapp/result_index.html',
        'result_detail': f'enrichapp/policyapp/result_detail.html',
        'workflow_index': f'enrichapp/policyapp/workflow_index.html',

    }
    self.select_columns_post_spec = {
        'list_columns': []
    }

api_policy_index(request, instancename)

Return the queryset

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("app_policy_index_api", nature="application")
def api_policy_index(self, request, instancename):
    """
    Return the queryset
    """
    from dashboard.apiv2 import lookup_specification

    spec = lookup_specification(name=instancename,
                                category=self.category)
    if spec is None:
        return {
            "status": "failure",
            "message": f"Unknown specification: {self.name}"
        }

    namespace = spec.get('namespace', 'default')
    PolicyModel    = self.get_model(spec, 'app_policy')

    qs = PolicyModel.objects.filter(active=True,
                                    appname=self.name,
                                    namespace=namespace)
    policies = [p.export(spec) for p in qs]

    return {
        "status": "success",
        "data": policies
    }

get_fs_handle(cred)

Get s3/gcs filesystem handle..

Source code in enrichsdk/app/bases/policyapp/views.py

def get_fs_handle(self, cred):
    """
    Get s3/gcs filesystem handle..
    """
    cred = get_credentials_by_name(cred)
    nature = cred['nature']
    if nature not in ['s3', 'gcs']:
        raise Exception(f"Unknown credentials: {nature}")

    if nature == 's3':
        config_kwargs={
            'signature_version': 's3v4'
        }

        if 'region' in cred:
            config_kwargs['region_name'] = cred['region']

        fshandle = s3fs.S3FileSystem(
            key    = cred['access_key'],
            secret = cred['secret_key'],
            config_kwargs=config_kwargs,
            use_listings_cache=False
        )
    else:
        fshandle = gcsfs.GCSFileSystem(
            token=cred['keyfile']
        )

    return fshandle

get_result(request, spec, policy)

Get data from s3 and other places

Source code in enrichsdk/app/bases/policyapp/views.py

def get_result(self, request, spec, policy):
    """
    Get data from s3 and other places
    """
    try:

        data = spec['data']
        handle = self.get_fs_handle(data['cred'])

        appname = self.name
        namespace = spec['namespace']

        root = os.path.join(data['root'], appname, namespace)

        # may be different from policy.name
        name = request.GET.get('name')

        fallback = spec.get('fallback',False)
        if fallback:
            if not handle.exists(root):
                messages.warning(request, "Result root directory is missing. choosing alternative")
                namespaces = handle.ls(os.path.join(data['root'], appname))
                root = namespaces[0]

        names = [os.path.basename(n) for n in handle.ls(root)]
        if name not in names:
            raise Exception(f"Could not find {name}")

        dates = sorted(handle.ls(os.path.join(root, name)), reverse=True)
        if len(dates) == 0:
            return None, None

        paths = sorted(handle.ls(dates[0]), reverse=True)
        if len(paths) == 0:
            return None

        content = None
        paths = [p for p in paths if not p.endswith('metadata.json')]
        for p in paths:
            try:
                content = self.get_one_result_path(request,
                                                   spec, policy,
                                                   handle, p)
                return p, content
            except:
                #traceback.print_exc()
                continue

        if hasattr(handle, 'close'):
            handle.close()

    except:
        logger.exception(f"Error in accessing {policy.name} results")

    return None, None

policy_add(request, spec)

Add a policy

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("app_policy_add", nature="application")
def policy_add(self, request, spec):
    """
    Add a policy
    """

    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    usecase = spec['usecase']
    template = self.get_template(spec, 'policy_add')

    # => Get models
    namespace   = spec.get('namespace','default')
    PolicyModel = self.get_model(spec, 'app_policy')
    PolicyForm  = self.get_form(spec,  'app_policy')

    config = request.GET.get('config', '{}')
    config = json.loads(config)

    if request.method != "POST":

        initial = {
            "config": config,
        }

        form = PolicyForm(initial=initial)

        return render(request,
                      template,
                      {
                          'app': self,
                          'spec': spec,
                          'usecase': usecase,
                          'basenamespace': r.namespace,
                          'form': form
                      })


    form = PolicyForm(request.POST)
    if form.is_valid():
        policy = form.save(commit=False)
        policy.appname = self.name
        policy.namespace = namespace
        policy.created_by = request.user
        policy.modified_by = request.user
        policy.active = True
        policy.save()

        log_app_activity(request, self.name, spec, "policy added", policy.desc)

        messages.success(request, 'Policy {} saved!'.format(policy.name))
        return HttpResponseRedirect(reverse(r.namespace + ":policy_index"))


    return render(request,
                  template,
                  {
                      'app': self,
                      'spec': spec,
                      'usecase': usecase,
                      'basenamespace': r.namespace,
                      'form': form
                  })

policy_generate_config(request, spec)

Process the select columns form

Source code in enrichsdk/app/bases/policyapp/views.py

def policy_generate_config(self, request, spec):
    """
    Process the select columns form
    """

    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    source_id = request.GET.get('source_id', None)
    if ((source_id is None) or (request.method != "POST")):
        logger.error("Missing source id or not a post")
        return HttpResponseRedirect(reverse(r.namespace + ":policy_select_dataset"))

    config = request.GET.dict()
    config.update(request.POST.dict())
    listcols = self.select_columns_post_spec['list_columns']
    for col in listcols:
        config[col] = request.POST.getlist(col)

    # Cleanup
    config.pop('csrfmiddlewaretoken',None)

    valid = self.validate_policy_config(request, spec, config)

    return valid, config

policy_index(request, spec)

Policy index...

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("app_policy_index", nature="application")
def policy_index(self, request, spec):
    """
    Policy index...
    """

    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    usecase = spec['usecase']

    template = self.get_template(spec, 'policy_index')

    namespace = spec.get('namespace', 'default')
    try:
        namepace = spec['namespace']
        PolicyModel    = self.get_model(spec, 'app_policy')
        policies = PolicyModel.objects\
                              .filter(appname=self.name, namespace=namespace)\
                              .order_by("-id")

    except:
        logger.exception(f"Unable to get {self.name} policies. Please see server log")
        #return HttpResponseRedirect(reverse(r.namespace + ":index"))

    return render(request,
                  template,
                  {
                      'app': self,
                      'spec': spec,
                      'usecase': usecase,
                      'basenamespace': r.namespace,
                      'policies': policies
                  })

policy_select_columns(request, spec)

Select columns

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("app_policy_select_columns", nature="application")
def policy_select_columns(self, request, spec):
    """
    Select columns
    """
    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    usecase = spec['usecase']
    template = self.get_template(spec, 'policy_select_columns')

    # Now access the metadata server..
    source_id = request.GET.get('source_id', None)
    if source_id is None:
        logger.error("Missing source id")
        return HttpResponseRedirect(reverse(r.namespace + ":policy_select_dataset"))
    source_version = request.GET.get('source_version', 'v1')

    # Handle the post from selection...
    if request.method == "POST":
        valid, config = self.policy_generate_config(request, spec)
        if not valid:
            # Errors got incorporated in the validate function..
            messages.error(request, "Invalid selection")
            return HttpResponseRedirect(reverse(r.namespace + ":policy_select_columns") + f"?source_id={source_id}&source_version={source_version}")
        try:
            config = json.dumps(config)
            return HttpResponseRedirect(reverse(r.namespace + ":policy_add") + f"?config={config}")
        except Exception as e:
            logger.exception("Error while redirecting to policy add")
            messages.error(request, "Error while adding a policy")
            return HttpResponseRedirect(reverse(r.namespace + ":policy_select_columns") + f"?source_id={source_id}&source_version={source_version}")


    # Collect the sources
    cred = spec['doodle']
    if callable(cred):
        cred = cred(spec)
    doodle = Doodle(cred)

    source = doodle.get_source(source_id)
    features = doodle.list_features(source_id=source['id'])

    # Now render
    return render(request,
                  template,
                  {
                      'app': self,
                      'spec': spec,
                      'usecase': usecase,
                      'basenamespace': r.namespace,
                      'select_columns_helper': self.get_template(spec, 'policy_select_columns_helper'),
                      'source': source,
                      'columns': features
              })

policy_select_dataset(request, spec)

Select datasets to process

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("app_policy_select_dataset", nature="application")
def policy_select_dataset(self, request, spec):
    """
    Select datasets to process
    """
    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    usecase = spec['usecase']

    template = self.get_template(spec, 'policy_select_dataset')
    cred = spec['doodle']
    if callable(cred):
        cred = cred(spec)

    doodle = Doodle(cred)

    # Collect the sources
    sources = doodle.list_sources()

    return render(request,
                  template,
                  {
                      'app': self,
                      'spec': spec,
                      'usecase': usecase,
                      'basenamespace': r.namespace,
                      'sources': sources
                  })

policy_toggle(request, spec, policy_id)

Activate/Deactivate policy

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("app_policy_toggle", nature="application")
def policy_toggle(self, request, spec, policy_id):
    """
    Activate/Deactivate policy
    """

    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    usecase = spec['usecase']


    namespace      = spec.get('namespace', 'default')
    PolicyModel    = self.get_model(spec, 'app_policy')
    policies       = PolicyModel.objects.filter(pk=policy_id, appname=self.name, namespace=namespace)
    if policies.count() == 0:
        messages.error(request, "No policy found")
        return HttpResponseRedirect(reverse(r.namespace + ":policy_index"))

    policy = policies[0]
    policy.active = not policy.active
    policy.save()
    messages.success(request, "Policy updated")
    return HttpResponseRedirect(reverse(r.namespace + ":policy_index"))

result_detail(request, spec)

Show the available results

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("app_result_detail", nature="application")
def result_detail(self, request, spec):
    """
    Show the available results
    """
    from dashboard.models import AppActionTarget

    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    fallback = spec.get('fallback', False)
    usecase = spec['usecase']

    template = self.get_template(spec, 'result_detail')

    name = request.GET.get('name', None)
    if name is None:
        messages.error(request, "Missing configuration")
        return HttpResponseRedirect(reverse(r.namespace + ":result_index"))

    try:
        namespace      = spec['namespace']
        PolicyModel    = self.get_model(spec, 'app_policy')
        policy         = PolicyModel.objects\
                                      .get(appname=self.name,
                                           namespace=namespace,
                                           name=name,
                                           active=True
                                      )
    except Exception as e:
        logger.exception(f"Unable to get configuration for {name}")
        if not fallback:
            messages.error(request, f"Unable to get configuration for {name}")
            return HttpResponseRedirect(reverse(r.namespace + ":result_index"))

        messages.warning(request, f"Unable to get configuration for {name}")
        try:
            policy = PolicyModel.objects.filter(appname=self.name,namespace=namespace, active=True).first()
        except:
            messages.warning(request, f"No policy defined for fallback")
            return HttpResponseRedirect(reverse(r.namespace + ":result_index"))

    path, data = self.get_result(request, spec, policy)
    if data is None:
        messages.error(request, f"Unable to get data for configuration {name}")
        logger.exception(f"Unable to get data for configuration {name}")
        return HttpResponseRedirect(reverse(r.namespace + ":result_index"))

    # Workflow targets
    supported_actions = self.get_supported_actions(request, spec, policy)
    targets = AppActionTarget.objects.filter(nature__in=supported_actions)

    return render(request,
                  template,
                  {
                      'app': self,
                      'spec': spec,
                      'usecase': usecase,
                      'basenamespace': r.namespace,
                      'name': name,
                      'policy': policy,
                      'data': data,
                      'path': path,
                      'targets': targets
                  })

result_index(request, spec)

Show the available results

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("app_result_index", nature="application")
def result_index(self, request, spec):
    """
    Show the available results
    """

    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    usecase = spec['usecase']

    template = self.get_template(spec, 'result_index')

    namespace = spec.get('namespace', 'default')
    try:
        namepace = spec['namespace']
        PolicyModel    = self.get_model(spec, 'app_policy')
        policies = PolicyModel.objects\
                              .filter(appname=self.name,
                                      namespace=namespace)\
                              .order_by("-id")
    except Exception as e:
        messages.error(request, f"Error! {e}")
        logger.exception(f"Unable to get {namespace} configurations. Please see server log")
        return HttpResponseRedirect(reverse(r.namespace + ":policy_index"))

    available = self.get_result_list(request, spec, policies)

    return render(request,
                  template,
                  {
                      'app': self,
                      'spec': spec,
                      'usecase': usecase,
                      'basenamespace': r.namespace,
                      'policies': policies,
                      'available': available
                  })

validate_policy_config(request, spec, config)

Validate the config generated by the backend

Source code in enrichsdk/app/bases/policyapp/views.py

def validate_policy_config(self, request, spec, config):
    """
    Validate the config generated by the backend
    """
    return len(config) > 0

workflow_action(request, spec)

Show the workflows for the given data

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("workflow_action", nature="application")
def workflow_action(self, request, spec):
    """
    Show the workflows for the given data
    """

    from dashboard.models import AppActionTarget

    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    usecase = spec['usecase']

    name = request.GET.get('name', None)
    if name is None:
        logger.exception("Missing name of the result")
        return HttpResponseRedirect(reverse(r.namespace + ":result_index"))

    data = request.POST.dict()

    try:
        try:
            target = AppActionTarget.objects.get(pk=data['actionid'])
        except:
            raise Exception("Internal error. Missing or invalid action id")

        messages.success(request, "Processed the post")
        status = "success"

        # Make this happen...
        status = self.take_action(request, spec, name, data, target)

    except Exception as e:
        return JsonResponse({
            'status': 'failure',
            'message': f"Failed to trigger action. {str(e)}"
        })

    return status

workflow_index(request, spec)

Show the workflows for the given data

Source code in enrichsdk/app/bases/policyapp/views.py

@log_activity("workflow_index", nature="application")
def workflow_index(self, request, spec):
    """
    Show the workflows for the given data
    """

    from dashboard.models import AppActionTarget


    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    usecase = spec['usecase']

    name = request.GET.get('name', None)
    if name is None:
        logger.exception("Missing name of the result")
        return HttpResponseRedirect(reverse(r.namespace + ":result_index"))

    data = request.GET.get('data', '{}')
    try:
        data = json.loads(data)
    except:
        logger.exception("Missing data dictionary required for workflows")
        return HttpResponseRedirect(reverse(r.namespace + ":result_index"))

    template = self.get_template(spec, 'workflow_index')

    targets = AppActionTarget.objects.filter(nature__in=self.get_supported_actions(request, spec, policy))

    return render(request,
                  template,
                  {
                      'app': self,
                      'spec': spec,
                      'usecase': usecase,
                      'basenamespace': r.namespace,
                      "name": name,
                      'data': data,
                      'targets': targets
                  })

singlepageapp

This is the base app setup for applications that have the following component:

policy interface (extensible) api pipeline (backend) result (could be arbitrary)

views

AppBase()

A shareable view across apps

Source code in enrichsdk/app/bases/singlepageapp/views.py

def __init__(self):
    self.name = "singlepageapp"
    self.verbose_name = "Single Page App"
    self.category = "singlepageapp"
    self.templates = {
        'index': f'enrichapp/singlepageapp/index.html',
        'helper': f'enrichapp/singlepageapp/helper.html',
        'workflow_index': f'enrichapp/policyapp/workflow_index.html',
    }

get_fs_handle(cred)

Get s3/gcs filesystem handle..

Source code in enrichsdk/app/bases/singlepageapp/views.py

def get_fs_handle(self, cred):
    """
    Get s3/gcs filesystem handle..
    """
    cred = get_credentials_by_name(cred)
    nature = cred['nature']
    if nature not in ['s3', 'gcs']:
        raise Exception(f"Unknown credentials: {nature}")

    if nature == 's3':
        config_kwargs={
            'signature_version': 's3v4'
        }

        if 'region' in cred:
            config_kwargs['region_name'] = cred['region']

        fshandle = s3fs.S3FileSystem(
            key    = cred['access_key'],
            secret = cred['secret_key'],
            config_kwargs=config_kwargs,
            use_listings_cache=False
        )
    else:
        fshandle = gcsfs.GCSFileSystem(
            token=cred['keyfile']
        )

    return fshandle

workflow_action(request, spec)

Show the workflows for the given data

Source code in enrichsdk/app/bases/singlepageapp/views.py

@log_activity("workflow_action", nature="application")
def workflow_action(self, request, spec):
    """
    Show the workflows for the given data
    """

    from dashboard.models import AppActionTarget

    r = resolve(request.path)

    valid, redirect = self.check_prerequisites(request, spec)
    if not valid:
        return redirect

    usecase = spec['usecase']

    data = request.POST.dict()

    try:
        try:
            target = AppActionTarget.objects.get(pk=data['actionid'])
        except:
            raise Exception("Internal error. Missing or invalid action id")

        # messages.success(request, "Processed the post")
        status = "success"

        # Make this happen...
        status = self.take_action(request, spec, data, target)

    except Exception as e:
        return JsonResponse({
            'status': 'failure',
            'message': f"Failed to trigger action. {str(e)}"
        })

    return status