diff --git a/Function/.gitkeep b/Function/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Function/Sharveprocein/.gitkeep b/Function/Sharveprocein/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Function/Sharveprocein/Readme.md b/Function/Sharveprocein/Readme.md new file mode 100644 index 0000000000000000000000000000000000000000..a4871b55394077e04e3cc49ad113ba10bc387e7b --- /dev/null +++ b/Function/Sharveprocein/Readme.md @@ -0,0 +1,53 @@ +# Sharveprocein Function Deployment Guide + +## Description +The files in this repository are used to deploy the Sharveprocein Function in a cluster. The function's purpose is to retrieve location and status information of Bluetooth detection sites from [VicRoads Data Exchange](https://data-exchange.vicroads.vic.gov.au/), format the obtained data, remove redundant information, and finally index it into Elasticsearch. + +## Deployment Steps +1. **Package Creation**: Package the files `Sharveprocein.py`, `requirements.txt`, and `build.sh` into a `.zip` file. + + ```bash + ( + cd fission/functions/sharveprocein + zip -r sharveprocein.zip . + mv sharveprocein.zip ../ + ) + + ``` +2. **Create Package**: Execute the following command in the terminal to create a package: + +```bash +fission package create --spec --sourcearchive ./sharveprocein.zip --env python --name sharveprocein --buildcmd './build.sh' + +``` + +3. **Create function**: Use the created package to create the function: + +```bash +fission function create --spec --name sharveprocein --pkg sharveprocein --env python --entrypoint "Sharveprocein.main" --configmap shared-data + +``` + +4. **Create HTTP Trigger**: Create an HTTP trigger for the function: + +```bash +fission route create --spec --url /sharveprocein --function sharveprocein --name sharveprocein --creatingress + +``` + +5. **Apply Specifications**: Apply the specifications to the cluster: + +```bash +fission spec apply --specdir fission/specs --wait + +``` + +## test +To test the route and trigger, use the following command: +```bash +curl "http://127.0.0.1:9090/sharveprocein" | jq '.' + +``` +This will generate the **URL: "http://127.0.0.1:9090/sharveprocein"**. + + diff --git a/Function/Sharveprocein/Sharveprocein.py b/Function/Sharveprocein/Sharveprocein.py new file mode 100644 index 0000000000000000000000000000000000000000..9aae36cbddff4d01b1048ffb55be5eea00cd651a --- /dev/null +++ b/Function/Sharveprocein/Sharveprocein.py @@ -0,0 +1,71 @@ +import json, datetime, requests +from flask import current_app +from elasticsearch8 import Elasticsearch + +def config(k): + with open(f'/configs/default/shared-data/{k}', 'r') as f: + return f.read() + +def main(): + #get the raw data by url + traffic_url = "https://data-exchange-api.vicroads.vic.gov.au/bluetooth_data/sites" + headers = { + "Cache-Control": "no-cache", + "Ocp-Apim-Subscription-Key": "257b936748224c4b84e8bad2af33ffbb" + } + response = requests.get(traffic_url, headers=headers) + if response.status_code == 200: + current_app.logger.info(f'Data retrieved successfully from API.') + data = response.json() + # processing the row data + observations = [] + for obs in data: + interval_start = obs["latest_stats"].get("interval_start") + if interval_start: + parsed_timestamp = datetime.datetime.fromisoformat(interval_start) + formatted_timestamp = parsed_timestamp.strftime("%Y-%m-%d %H:%M:%S") + else: + formatted_timestamp = None + observation = { + "id": obs.get("id"), + "number": obs.get("number"), + "organization": { + "id": obs["organization"].get("id") + }, + "enabled": obs.get("enabled"), + "name": obs.get("name"), + "location": obs.get("location"), + "latest_stats": { + "interval_start": formatted_timestamp + } + } + observations.append(observation) + # exchange the result as JSON object + data_up = { + "observations": observations + } + # Connect to Elasticsearch + # Get Elasticsearch username and password from environment variables + es_username = config('ES_USERNAME') + es_password = config('ES_PASSWORD') + + # Connect to Elasticsearch + client = Elasticsearch( + 'https://elasticsearch-master.elastic.svc.cluster.local:9200', + verify_certs=False, + ssl_show_warn=False, + basic_auth=(es_username, es_password) # Use environment variables + ) + + # Log the data to be added to Elasticsearch + current_app.logger.info(f'Sites Data to add to Elasticsearch: {data}') + + # Index the data into Elasticsearch + res = client.index( + index='sites', # Replace with your index name + body=data_up + ) + # Log the result of indexing + current_app.logger.info('Indexed data successfully') + + return 'ok' \ No newline at end of file diff --git a/Function/Sharveprocein/build.sh b/Function/Sharveprocein/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..c9e6acd5b6c9075df9c812f81147801a7412f6c1 --- /dev/null +++ b/Function/Sharveprocein/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +pip3 install -r ${SRC_PKG}/requirements.txt -t ${SRC_PKG} && cp -r ${SRC_PKG} ${DEPLOY_PKG} diff --git a/Function/Sharveprocein/requirements.txt b/Function/Sharveprocein/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c865379996546bbe1d68d361d9ca9ac2b6700411 --- /dev/null +++ b/Function/Sharveprocein/requirements.txt @@ -0,0 +1 @@ +elasticsearch8==8.11.0 diff --git a/specs/.gitkeep b/specs/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/specs/README b/specs/README new file mode 100644 index 0000000000000000000000000000000000000000..1db3f9a5947a9b92420e928247772b36609b3277 --- /dev/null +++ b/specs/README @@ -0,0 +1,42 @@ + +Fission Specs +============= + +This is a set of specifications for a Fission app. This includes functions, +environments, and triggers; we collectively call these things "resources". + +How to use these specs +---------------------- + +These specs are handled with the 'fission spec' command. See 'fission spec --help'. + +'fission spec apply' will "apply" all resources specified in this directory to your +cluster. That means it checks what resources exist on your cluster, what resources are +specified in the specs directory, and reconciles the difference by creating, updating or +deleting resources on the cluster. + +'fission spec apply' will also package up your source code (or compiled binaries) and +upload the archives to the cluster if needed. It uses 'ArchiveUploadSpec' resources in +this directory to figure out which files to archive. + +You can use 'fission spec apply --watch' to watch for file changes and continuously keep +the cluster updated. + +You can add YAMLs to this directory by writing them manually, but it's easier to generate +them. Use 'fission function create --spec' to generate a function spec, +'fission environment create --spec' to generate an environment spec, and so on. + +You can edit any of the files in this directory, except 'fission-deployment-config.yaml', +which contains a UID that you should never change. To apply your changes simply use +'fission spec apply'. + +fission-deployment-config.yaml +------------------------------ + +fission-deployment-config.yaml contains a UID. This UID is what fission uses to correlate +resources on the cluster to resources in this directory. + +All resources created by 'fission spec apply' are annotated with this UID. Resources on +the cluster that are _not_ annotated with this UID are never modified or deleted by +fission. + diff --git a/specs/env-nodejs.yaml b/specs/env-nodejs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5d5288ca0ee459bf0b892a16a09eeae94322b45 --- /dev/null +++ b/specs/env-nodejs.yaml @@ -0,0 +1,22 @@ +apiVersion: fission.io/v1 +kind: Environment +metadata: + creationTimestamp: null + name: nodejs +spec: + builder: + command: build + container: + name: "" + resources: {} + image: fission/node-builder + imagepullsecret: "" + keeparchive: false + poolsize: 3 + resources: {} + runtime: + container: + name: "" + resources: {} + image: fission/node-env + version: 2 diff --git a/specs/fission-deployment-config.yaml b/specs/fission-deployment-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bae404827563fde61fafc519e9f0e76a48aceced --- /dev/null +++ b/specs/fission-deployment-config.yaml @@ -0,0 +1,7 @@ +# This file is generated by the 'fission spec init' command. +# See the README in this directory for background and usage information. +# Do not edit the UID below: that will break 'fission spec apply' +apiVersion: fission.io/v1 +kind: DeploymentConfig +name: fission +uid: 4652106a-735e-49fa-9f83-7867026168e6 diff --git a/specs/function-sharveprocein.yaml b/specs/function-sharveprocein.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba648a298f3b70b41d157e5501d6d1d1547849c2 --- /dev/null +++ b/specs/function-sharveprocein.yaml @@ -0,0 +1,30 @@ +apiVersion: fission.io/v1 +kind: Function +metadata: + creationTimestamp: null + name: sharveprocein +spec: + InvokeStrategy: + ExecutionStrategy: + ExecutorType: poolmgr + MaxScale: 0 + MinScale: 0 + SpecializationTimeout: 120 + TargetCPUPercent: 0 + StrategyType: execution + concurrency: 500 + configmaps: + - name: shared-data + namespace: "" + environment: + name: python + namespace: "" + functionTimeout: 60 + idletimeout: 120 + package: + functionName: Sharveprocein.main + packageref: + name: sharveprocein + namespace: "" + requestsPerPod: 1 + resources: {} diff --git a/specs/package-sharveprocein.yaml b/specs/package-sharveprocein.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22adb131a5e69cb0be6eb90ad8333dae98ea2514 --- /dev/null +++ b/specs/package-sharveprocein.yaml @@ -0,0 +1,25 @@ +include: +- ./sharveprocein.zip +kind: ArchiveUploadSpec +name: sharveprocein-zip-eCNp + +--- +apiVersion: fission.io/v1 +kind: Package +metadata: + creationTimestamp: null + name: sharveprocein +spec: + buildcmd: ./build.sh + deployment: + checksum: {} + environment: + name: python + namespace: "" + source: + checksum: {} + type: url + url: archive://sharveprocein-zip-eCNp +status: + buildstatus: pending + lastUpdateTimestamp: "2024-05-09T12:44:11Z" diff --git a/specs/route-sharveprocein.yaml b/specs/route-sharveprocein.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e40b38d8f2b8e87911e12396722413683745781 --- /dev/null +++ b/specs/route-sharveprocein.yaml @@ -0,0 +1,22 @@ +apiVersion: fission.io/v1 +kind: HTTPTrigger +metadata: + creationTimestamp: null + name: sharveprocein +spec: + createingress: true + functionref: + functionweights: null + name: sharveprocein + type: name + host: "" + ingressconfig: + annotations: null + host: '*' + path: /sharveprocein + tls: "" + method: "" + methods: + - GET + prefix: "" + relativeurl: /sharveprocein diff --git a/specs/shared-data.yaml b/specs/shared-data.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7e5ce6aed1ef0446a64061cb5d914b9dffa8c39 --- /dev/null +++ b/specs/shared-data.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: default + name: shared-data +data: + ES_USERNAME: elastic + ES_PASSWORD: elastic