diff --git a/.env b/.env new file mode 100644 index 00000000..dad48d80 --- /dev/null +++ b/.env @@ -0,0 +1,14 @@ +# for local development only +# usually this file should not be included in the repository + +MYSQL_ROOT_PASSWORD=rootPassword1 +MYSQL_DATABASE=financial +MYSQL_USER=ritheesh +MYSQL_PASSWORD=ritheeshPassword1 +MYSQL_PORT=3306 +DB_PORT=3306 +DB_HOST=127.0.0.1 +DB_NAME=financial +DB_USER=ritheesh +DB_PASSWORD=ritheeshPassword1 +ALPHAVANTAGE_API_KEY=15SWOEC7H3CLW3B1 diff --git a/.github/workflows/publish-docker-image.yaml b/.github/workflows/publish-docker-image.yaml new file mode 100644 index 00000000..2f96a1d7 --- /dev/null +++ b/.github/workflows/publish-docker-image.yaml @@ -0,0 +1,46 @@ +name: Build and Publish Docker Image + +on: + push: + branches: + - main + - master + +env: + IMAGE_TAG: latest + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Build Docker image + run: | + docker-compose build + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Push the Docker image to Docker Hub + run: | + echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin + docker tag python_assignment_web:latest "${{ secrets.DOCKER_USERNAME }}/financial-web:${IMAGE_TAG}" + docker push "${{ secrets.DOCKER_USERNAME }}/financial-web:${IMAGE_TAG}" + + # - name: Upload docker image + # uses: actions/upload-artifact@v2 + # with: + # name: docker images for financial servers + # path: | + # python_assignment_web + + # - name: Download docker image + # uses: actions/download-artifact@v2 + # with: + # name: python_assignment_web diff --git a/.github/workflows/test-financial-data.yaml b/.github/workflows/test-financial-data.yaml new file mode 100644 index 00000000..64da3dd0 --- /dev/null +++ b/.github/workflows/test-financial-data.yaml @@ -0,0 +1,39 @@ +name: Test Financial Data APIs + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Apply migrations + run: | + export TEST_DATABASE=ON + python financial/manage.py makemigrations + python financial/manage.py migrate + + - name: Run web server tests + run: | + export TEST_DATABASE=ON + python financial/manage.py test core.tests + + - name: Run tests for get_raw_data + run: | + python -m unittest tests/test_get_raw_data.py diff --git a/.github/workflows/test-web-db-server.yaml b/.github/workflows/test-web-db-server.yaml new file mode 100644 index 00000000..a9b39131 --- /dev/null +++ b/.github/workflows/test-web-db-server.yaml @@ -0,0 +1,40 @@ +name: Test Web & Db Servers + +on: + push: + branches: + - main + - master + +jobs: + test: + runs-on: ubuntu-latest + + # services: + # db: + # image: mysql:latest + # env: + # MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD } + # MYSQL_DATABASE: ${{ secrets.MYSQL_DATABASE }} + # MYSQL_USER: ${{ secrets.MYSQL_USER }} + # MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }} + # ports: + # - 3306:3306 + # options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Start containers + run: | + docker-compose up -d + sleep 10 + docker ps + + # - name: Test database server + # run: docker-compose exec db mysql -h localhost -u ${MYSQL_USER} -p ${MYSQL_PASSWORD} ${MYSQL_DATABASE} -e "SELECT 1" || exit 1 + + # - name: Test web server + # run: | + # curl -f http://localhost:5000/ || exit 1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e16c90a5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,141 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# custom for financial_data app +.vscode/ +*.db +*.log +_* +__pycache__/ +*.pyc +financial/core/migrations/ + +data/** +init.sql +*.tar diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c4771c7b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# Start from a base image of Python 3.8 +FROM python:3.8 + +# Set environment variables +# PYTHONDONTWRITEBYTECODE: Prevents Python from writing .pyc files to disk (reduces clutter) +# PYTHONUNBUFFERED: Ensures Python output is sent straight to terminal (useful for logging) +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +# Set the working directory to /app +WORKDIR /app + +# Copy the requirements file into the container +COPY requirements.txt . + +# Install any needed packages specified in requirements.txt +RUN pip install --no-cache-dir -r requirements.txt -v + +# Copy the current directory contents into the container at /app +COPY . /app + +# Expose port 3306, 5000 for the MySQL server and financial django server +EXPOSE 5000 +EXPOSE 3306 diff --git a/README.md b/README.md index 210302c3..ebb09c8d 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,177 @@ -# Take-Home Assignment +# Financial Data Retrieval Application -The goal of this take-home assignment is to evaluate your abilities to use API, data processing and transformation, SQL, and implement a new API service in Python. +![Python](https://img.shields.io/badge/Python-3.8-blue?style=flat&logo=python) +![Django](https://img.shields.io/badge/Django-4.1-green?style=flat&logo=django) +![MySQL](https://img.shields.io/badge/MySQL-8.0.32-blue?style=flat&logo=mysql) +![Docker](https://img.shields.io/badge/Docker-20.10.9-blue?style=flat&logo=docker) +![REST API](https://img.shields.io/badge/REST%20API-Django%20REST%20Framework-green) -You should first fork this repository, and then send us the code or the url of your forked repository via email. -**Please do not submit any pull requests to this repository.** +`Financial Data API` project is a simple REST API built using Django Rest Framework that provides financial data to users. The API allows users to retrieve financial data of two stocks (IBM, Apple Inc.) for the most recently two weeks. The data is sourced from the [AlphaVantage](https://www.alphavantage.co/documentation/) API, which requires an API key to access the data. The API provides endpoints for retrieving raw data as well as endpoints for retrieving pre-processed data. -You need to perform the following **Two** tasks: +[GitHub Page](https://ritheeshbaradwaj.github.io/python_assignment/) -## Task1 -### Problem Statement: -1. Retrieve the financial data of Two given stocks (IBM, Apple Inc.)for the most recently two weeks. Please using an free API provider named [AlphaVantage](https://www.alphavantage.co/documentation/) -2. Process the raw API data response, a sample output after process should be like: +## Status + +[![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) + +[![Build and Publish Docker Image](https://github.com/RitheeshBaradwaj/python_assignment/actions/workflows/publish-docker-image.yaml/badge.svg?branch=main)](https://github.com/RitheeshBaradwaj/python_assignment/actions/workflows/publish-docker-image.yaml) + +[![Test Financial Data APIs](https://github.com/RitheeshBaradwaj/python_assignment/actions/workflows/test-financial-data.yaml/badge.svg?branch=main)](https://github.com/RitheeshBaradwaj/python_assignment/actions/workflows/test-financial-data.yaml) + +[![Test Web & Db Servers](https://github.com/RitheeshBaradwaj/python_assignment/actions/workflows/test-web-db-server.yaml/badge.svg?branch=main)](https://github.com/RitheeshBaradwaj/python_assignment/actions/workflows/test-web-db-server.yaml) + +[Docker Image: financial-web](https://hub.docker.com/r/ritheeshbaradwaj/financial-web) + +## Tech Stack + +The tech stack used in this project is as follows: + +- Python 3.8: A high-level programming language commonly used for web development and data analysis +- Django 4.1: Django is a high-level Python web framework that enables rapid development of secure and maintainable websites +- Django Rest Framework 3.4: A web framework for building RESTful APIs with Django, a Python-based web framework +- MySQL 8.0.32: An open-source relational database management system (RDBMS) that is commonly used for web applications +- Docker: A containerization platform that allows for the easy creation, deployment, and management of applications in containers. + +## Installation Guide + +### Prerequisites + +- [Python 3.8.x](https://www.python.org/downloads/) +- [pip](https://pip.pypa.io/en/stable/installation/) +- [Docker](https://docs.docker.com/engine/install/) +- [MySQL](https://ubuntu.com/server/docs/databases-mysql) + +### Install Docker + +You can install Docker using the following commands: + +```bash +sudo apt-get update +sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common -y +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +sudo apt-get update +sudo apt-get install docker-ce docker-ce-cli containerd.io -y ``` -{ - "symbol": "IBM", - "date": "2023-02-14", - "open_price": "153.08", - "close_price": "154.52", - "volume": "62199013", -}, -{ - "symbol": "IBM", - "date": "2023-02-13", - "open_price": "153.08", - "close_price": "154.52", - "volume": "59099013" -}, -{ - "symbol": "IBM", - "date": "2023-02-12", - "open_price": "153.08", - "close_price": "154.52", - "volume": "42399013" -}, -... -``` -3. Insert the records above into a table named `financial_data` in your local database, column name should be same as the processed data from step 2 above (symbol, date, open_price, close_price, volume) - - -## Task2 -### Problem Statement: -1. Implement an Get financial_data API to retrieve records from `financial_data` table, please note that: - - the endpoint should accept following parameters: start_date, end_date, symbol, all parameters are optional - - the endpoint should support pagination with parameter: limit and page, if no parameters are given, default limit for one page is 5 - - the endpoint should return an result with three properties: - - data: an array includes actual results - - pagination: handle pagination with four properties - - - count: count of all records without panigation - - page: current page index - - limit: limit of records can be retrieved for single page - - pages: total number of pages - - info: includes any error info if applies - - -Sample Request: + +### Install Docker Compose + +You can install Docker Compose using pip: + ```bash -curl -X GET 'http://localhost:5000/api/financial_data?start_date=2023-01-01&end_date=2023-01-14&symbol=IBM&limit=3&page=2' +sudo apt-get update +sudo apt-get install python3-pip -y +pip3 install docker-compose +``` + +### Start Docker + +Start the Docker daemon using the following command: + +```bash +sudo service docker start +``` + +## Run Financal Data API With Docker Compose + +1. Clone the repository: + +```bash +git clone https://github.com/RitheeshBaradwaj/python_assignment.git +``` + +2. Install all prerequisites mentioned above (Python3, Docker). + +3. Navigate to the cloned repository directory. +```bash +cd python_assignment +``` + +4. Create a virtual environment. + +```bash +python3 -m venv financial_env +``` + +5. Activate the virtual environment. + +```bash +source financial_env/bin/activate ``` -Sample Response: + +6. Install the required packages using pip. + +```shell +pip install -r requirements.txt ``` + +7. For local developement, set all the env variables in `.env` + +8. Start mysql server and django server with docker-compose + +```bash +docker-compose up -d +``` + +Now, both web server and database servers are running, we need to populate the database `financial` with financial data +taken from AlphaVantage API. + +## Maintaining the API Key + +- In a development environment, you can store the API key as an environment variable, set `ALPHAVANTAGE_API_KEY` env var in `.env` file. + +- In a production environment, you should store the API key as a secure environment variable on your server (eg: GitHub secrity keys). You can claim a [free API](https://www.alphavantage.co/support/#api-key). + +For workflows, I stored the secerets here: [GitHub secerets](https://github.com/RitheeshBaradwaj/python_assignment/settings/secrets/actions) + +Run `get_raw_data.py` to populate the database with data from IBM, AAPL. + +```bash +python get_raw_data.py +``` + +## API Usage + +Once our database has some records and we can retrive them. + +The project provides the following APIs: + +- `/api/financial_data/`: Returns a list of financial data records. +- `/api/statistics/`: Returns statistics for financial data records. + +### /api/financial_data + +This API returns a list of financial data based on the given parameters. + +#### Request + +The following parameters are supported: + +- `start_date`: The start date for the financial data (optional). +- `end_date`: The end date for the financial data (optional). +- `symbol`: The stock symbol for the financial data (optional). +- `limit`: The number of items to return per page (optional). +- `page`: The page number to return (optional). + +#### Example request: + +```bash +curl -X GET 'http://localhost:5000/api/financial_data?start_date=2023-01-01&end_date=2023-01-14&symbol=IBM&limit=3&page=2' +``` + +#### Response + +The response will be a JSON object with the following keys: + +- `data`: An array of financial data objects. +- `pagination`: An object containing pagination information. +- `info`: An object containing additional information about the request, such as error messages. + +#### Example response: + +```bash { "data": [ { @@ -94,26 +204,36 @@ Sample Response: }, "info": {'error': ''} } - ``` -2. Implement an Get statistics API to perform the following calculations on the data in given period of time: - - Calculate the average daily open price for the period - - Calculate the average daily closing price for the period - - Calculate the average daily volume for the period +### /api/statistics - - the endpoint should accept following parameters: start_date, end_date, symbols, all parameters are required - - the endpoint should return an result with two properties: - - data: calculated statistic results - - info: includes any error info if applies +This API returns statistics for the financial data based on the given parameters. -Sample request: -```bash -curl -X GET http://localhost:5000/api/statistics?start_date=2023-01-01&end_date=2023-01-31&symbol=IBM +#### Request +The following parameters are supported: + +- `start_date`: The start date for the financial data (required). +- `end_date`: The end date for the financial data (required). +- `symbol`: The stock symbol for the financial data (required). + +#### Example request + +```bash +curl -X GET 'http://localhost:5000/api/statistics?start_date=2023-01-01&end_date=2023-01-31&symbol=IBM' ``` -Sample response: -``` + +#### Response + +The response will be a JSON object with the following keys: + +- `data`: An object containing statistics information. +- `info`: An object containing additional information about the request, such as error messages. + +#### Example response + +```bash { "data": { "start_date": "2023-01-01", @@ -125,104 +245,149 @@ Sample response: }, "info": {'error': ''} } +``` + +## Run Database And Web Server on Local Environment + +To run the database and webserver locally, you can follow below steps + +### Run Django +1. Install dependencies + +```bash +pip install -r requirements.txt +`` + +2. Run the migrations + +```bash +python financial/manage.py makemigrations +python financial/manage.py migrate ``` -## What you should deliver: -Directory structure: +3. Start the server at requried port + +```bash +python financial/manage.py runserver 0.0.0.0:5000 ``` -project-name/ -├── model.py -├── schema.sql -├── get_raw_data.py -├── Dockerfile -├── docker-compose.yml -├── README.md -├── requirements.txt -└── financial/ +### Run MySQL + +1. Install mysql server + +```bash +sudo apt-get update +sudo apt-get install mysql-server +sudo systemctl start mysql +sudo systemctl status mysql +``` + +2. To secure the installation, run the following command: + +```bash +sudo mysql_secure_installation +``` +This will guide you through a series of prompts to secure your installation. + +You can then log in to the MySQL server using the following command: + +```bash +sudo mysql -u root -p +``` + +Enter the root password you set during installation. + +To create a new database, use the following command: + +```bash +CREATE DATABASE dbname; +``` + +Replace "dbname" with the name you want to give your database. For our application we use `financial`. + +To create a new user and grant permissions to the database, use the following commands: + +```bash +CREATE USER 'username'@'localhost' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON dbname.* TO 'username'@'localhost'; ``` -1. A `get_raw_data.py` file in root folder +Replace "username" with the name you want to give your user and "password" with the password you want to use. - Action: - - Run - ```bash - python get_raw_data.py - ``` +## Running Tests - Expectation: - - 1. Financial data will be retrieved from API and processed,then insert all processed records into table `financial_data` in local db - 2. Duplicated records should be avoided when executing get_raw_data multiple times, consider implementing your own logic to perform upsert operation if the database you select does not have native support for such operation. +To run tests for this project, follow these steps: -2. A `schema.sql` file in root folder - - Define schema for financial_data table, if you prefer to use an ORM library, just **ignore** this deliver item and jump to item3 below. +1. Ensure that all the necessary dependencies are installed by running the command pip install -r requirements.txt - Action: Run schema definition in local db +### Getting Raw Data - Expectation: A new table named `financial_data` should be created if not exists in db +1. Execute the following command to run unit tests for `get_raw_data.py` -3. (Optional) A `model.py` file: - - If you perfer to use a ORM library instead of DDL, please include your model definition in `model.py`, and describe how to perform migration in README.md file +```bash +python -m unittest tests/test_get_raw_data.py +`` -4. A `Dockerfile` file in root folder +2. To test `get_raw_data.py with sql server. Start the test database and execute following command - Build up your local API service +```bash +python -m unittest tests/test_get_raw_data.py +``` -5. A `docker-compose.yml` file in root folder +### Django Tests For APIs - Two services should be defined in docker-compose.yml: Database and your API +1. Run the following command to run migrations for test database - Action: +```bash +export TEST_DATABASE=ON +python financial/manage.py makemigrations +python financial/manage.py migrate +``` - ```bash - docker-compose up - ``` +2. Run the tests for `core` app - Expectation: - Both database and your API service is up and running in local development environment +```bash +python financial/manage.py test core.tests +``` + +The tests will run and output the results to the console. Any failures or errors will also be displayed along with the traceback for easy debugging. + +## How to Check the Published Docker Image + +I have used GitHub Actions to publish the latest docker image to Docker Hub. +To check the published Docker image, you can run the following command in your terminal after logging into Docker Hub: -6. A `financial` sub-folder: +Image link: https://hub.docker.com/r/ritheeshbaradwaj/financial-web - Put all API implementation related codes in here +```bash +docker pull ritheeshbaradwaj/financial-web +``` -7. `README.md`: +To verify that the image has been pulled successfully, you can run the following command: - You should include: - - A brief project description - - Tech stack you are using in this project - - How to run your code in local environment - - Provide a description of how to maintain the API key to retrieve financial data from AlphaVantage in both local development and production environment. +```bash +docker images +``` -8. A `requirements.txt` file: +This will display a list of all the images you have downloaded to your machine. Check if the image you just downloaded is listed there. - It should contain your dependency libraries. +## How to Check the GitHub Workflows -## Requirements: +To check the GitHub Workflows, follow these steps: -- The program should be written in Python 3. -- You are free to use any API and libraries you like, but should include a brief explanation of why you chose the API and libraries you used in README. -- The API key to retrieve financial data should be stored securely. Please provide a description of how to maintain the API key from both local development and production environment in README. -- The database in Problem Statement 1 could be created using SQLite/MySQL/.. with your own choice. -- The program should include error handling to handle cases where the API returns an error or the data is not in the correct format. -- The program should cover as many edge cases as possible, not limited to expectations from deliverable above. -- The program should use appropriate data structures and algorithms to store the data and perform the calculations. -- The program should include appropriate documentation, including docstrings and inline comments to explain the code. +1. Click on the "Actions" tab +2. Here you can see a list of workflows that have been set up in this repository -## Evaluation Criteria: +- `Test Financial Data APIs`: To test `get_raw_data.py` and django `financial/core` apis +- `Build and Publish Docker Image`: To build and publish latest docker image to Docker Hub -Your solution will be evaluated based on the following criteria: +## Contact Information -- Correctness: Does the program produce the correct results? -- Code quality: Is the code well-structured, easy to read, and maintainable? -- Design: Does the program make good use of functions, data structures, algorithms, databases, and libraries? -- Error handling: Does the program handle errors and unexpected input appropriately? -- Documentation: Is the code adequately documented, with clear explanations of the algorithms and data structures used? +I appreciate your interest in `Financial Data Retrieval APIs`. You can reach to me through the following channels: -## Additional Notes: +- Email: ritheeshbaradwaj@gmail.com +- LinkedIn: [ritheesh-baradwaj-yellenki](https://www.linkedin.com/in/ritheesh-baradwaj-yellenki/) +- [GitHub Issues](https://github.com/RitheeshBaradwaj/python_assignment/issues) -You have 7 days to complete this assignment and submit your solution. +## Thank you :D diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..cd76c9a7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +version: "3" + +services: + web: + build: . + command: > + bash -c ' + python financial/manage.py makemigrations + python financial/manage.py migrate + python financial/manage.py runserver 0.0.0.0:5000' + volumes: + - .:/app + ports: + - "5000:5000" + environment: + DB_HOST: db + DB_NAME: ${MYSQL_DATABASE} + DB_USER: ${MYSQL_USER} + DB_PASSWORD: ${MYSQL_PASSWORD} + depends_on: + - db + healthcheck: + # Test the health endpoint + test: ["CMD", "curl", "-f", "http://localhost:5000/"] + interval: 30s + timeout: 10s + retries: 3 + + db: + image: mysql:latest + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} + MYSQL_DATABASE: ${MYSQL_DATABASE} + MYSQL_USER: ${MYSQL_USER} + MYSQL_PASSWORD: ${MYSQL_PASSWORD} + ports: + - "3306:3306" + volumes: + - ./data:/var/lib/mysql + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 10s + timeout: 5s + retries: 3 + # command: > + # bash -c ' + # until echo "SELECT 1" | mysql -hdb -uroot -p"$MYSQL_ROOT_PASSWORD" &> /dev/null + # do + # >&2 echo "MySQL is unavailable - sleeping" + # sleep 1 + # done + # >&2 echo "MySQL is up - executing command" + # mysql -hdb -uroot -p"$MYSQL_ROOT_PASSWORD" -e "ALTER USER root@localhost IDENTIFIED WITH mysql_native_password BY '\''${MYSQL_ROOT_PASSWORD}'\'';"' diff --git a/financial/core/admin.py b/financial/core/admin.py new file mode 100644 index 00000000..b8891cc5 --- /dev/null +++ b/financial/core/admin.py @@ -0,0 +1,4 @@ +from django.contrib import admin +from .models import FinancialDataModel + +admin.site.register(FinancialDataModel) diff --git a/financial/core/apps.py b/financial/core/apps.py new file mode 100644 index 00000000..c0ce093b --- /dev/null +++ b/financial/core/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CoreConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "core" diff --git a/financial/core/models.py b/financial/core/models.py new file mode 100644 index 00000000..76f72f77 --- /dev/null +++ b/financial/core/models.py @@ -0,0 +1,15 @@ +from django.db import models + + +class FinancialDataModel(models.Model): + symbol = models.CharField(max_length=20) + date = models.DateField() + open_price = models.DecimalField(max_digits=20, decimal_places=2) + close_price = models.DecimalField(max_digits=20, decimal_places=2) + volume = models.BigIntegerField() + + def __str__(self): + return f"{self.symbol} - {self.date}" + + class Meta: + db_table = "financial_data" diff --git a/financial/core/serializers.py b/financial/core/serializers.py new file mode 100644 index 00000000..c1e3849c --- /dev/null +++ b/financial/core/serializers.py @@ -0,0 +1,8 @@ +from rest_framework import serializers +from .models import FinancialDataModel + + +class FinancialDataSerializer(serializers.ModelSerializer): + class Meta: + model = FinancialDataModel + fields = ("id", "symbol", "date", "open_price", "close_price", "volume") diff --git a/financial/core/tests.py b/financial/core/tests.py new file mode 100644 index 00000000..a03c4a51 --- /dev/null +++ b/financial/core/tests.py @@ -0,0 +1,430 @@ +from rest_framework import status +from rest_framework.test import APIClient + +from .models import FinancialDataModel +from django.test import TestCase +from django.urls import reverse +from django.utils import timezone + +from unittest.mock import Mock, patch +from datetime import datetime, timedelta + + +class FinancialDataModelTest(TestCase): + def setUp(self): + self.client = APIClient() + self.start_date = datetime(2022, 1, 1).strftime("%Y-%m-%d") + self.end_date = datetime(2022, 1, 31).strftime("%Y-%m-%d") + self.symbol = "AAPL" + self.url = reverse("statistics") + + def test_get_statistics_api_view(self): + # Arrange + # Create a mock queryset for the FinancialDataModel + queryset_mock = Mock() + queryset_mock.filter.return_value = queryset_mock + queryset_mock.aggregate.return_value = { + "open_price__avg": 100.0, + "close_price__avg": 110.0, + "volume__sum": 1000000, + } + + # Patch the FinancialDataModel.objects.filter method to return the mock queryset + with patch.object( + FinancialDataModel.objects, "filter", return_value=queryset_mock + ): + # Act + # Make a GET request to the StatisticsAPIView + response = self.client.get( + self.url, + { + "start_date": self.start_date, + "end_date": self.end_date, + "symbol": self.symbol, + }, + format="json", + ) + + # Assert + # Check that the response status code is 200 OK + self.assertEqual(response.status_code, status.HTTP_200_OK) + + # Check that the response data matches the expected data + expected_data = { + "data": { + "start_date": datetime.strptime(self.start_date, "%Y-%m-%d"), + "end_date": datetime.strptime(self.end_date, "%Y-%m-%d"), + "symbol": self.symbol, + "average_daily_open_price": 100.0, + "average_daily_close_price": 110.0, + "average_daily_volume": 1000000, + }, + "info": {"error": ""}, + } + self.assertEqual(response.data, expected_data) + + def test_get_statistics_api_view_missing_parameters(self): + # Arrange & Act + # Make a GET request to the StatisticsAPIView with missing parameters + response = self.client.get(self.url, {}, format="json") + + # Assert + # Check that the response status code is 400 Bad Request + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + # Check that the response data matches the expected data + expected_data = { + "data": {}, + "info": { + "error": "start_date, end_date, and symbol are required parameters" + }, + } + self.assertEqual(response.data, expected_data) + + def test_get_statistics_api_view_invalid_date_format(self): + # Arrange & Act + # Make a GET request to the StatisticsAPIView with invalid date format + response = self.client.get( + self.url, + { + "start_date": "2022-01-01", + "end_date": "2022/01/31", + "symbol": self.symbol, + }, + format="json", + ) + + # Assert + # Check that the response status code is 400 Bad Request + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + # Check that the response data matches the expected data + expected_data = { + "data": {}, + "info": { + "error": "start_date and end_date must be in the format YYYY-MM-DD" + }, + } + self.assertEqual(response.data, expected_data) + + def test_get_statistics_api_view_invalid_start_end_date(self): + # Arrange & Act + # Make a GET request to the StatisticsAPIView with invalid date format + response = self.client.get( + self.url, + { + "start_date": "2022-01-01", + "end_date": "2021-01-01", + "symbol": self.symbol, + }, + format="json", + ) + + # Assert + # Check that the response status code is 400 Bad Request + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + # Check that the response data matches the expected data + expected_data = { + "data": {}, + "info": {"error": "start_date must be before end_date"}, + } + self.assertEqual(response.data, expected_data) + + @patch("core.models.FinancialDataModel.objects.filter") + def test_filter_queryset_by_required_parameters(self, mock_filter): + # Arrange + # Test that the view filters the queryset by the required parameters + start_date = (timezone.now() - timedelta(days=2)).date().strftime("%Y-%m-%d") + end_date = timezone.now().date().strftime("%Y-%m-%d") + symbol = "TEST1" + + mock_filter.return_value = FinancialDataModel.objects.none() + + # Act + # Make a GET request to the StatisticsAPIView + response = self.client.get( + self.url, {"start_date": start_date, "end_date": end_date, "symbol": symbol} + ) + + # Act + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["data"]), 6) + self.assertIsNone(response.data["data"]["average_daily_open_price"]) + self.assertIsNone(response.data["data"]["average_daily_close_price"]) + self.assertIsNone(response.data["data"]["average_daily_volume"]) + + +from django.test import TestCase, RequestFactory +from unittest.mock import patch, Mock +from unittest import mock +from datetime import datetime +from rest_framework import status + +from .views import FinancialDataAPIView +from .serializers import FinancialDataSerializer +from .models import FinancialDataModel +from django.http import HttpRequest +from django.http import QueryDict + + +class FinancialDataAPIViewTestCase(TestCase): + def setUp(self): + self.factory = RequestFactory() + self.view = FinancialDataAPIView.as_view() + self.serializer_class = FinancialDataSerializer + self.queryset = FinancialDataModel.objects.none() + self.url = reverse("financial_data") + self.financial_data = FinancialDataModel.objects.create( + date="2022-01-01", + symbol="AAPL", + open_price=100.0, + close_price=95.0, + volume=1000000, + ) + + def test_get_queryset_symbol(self): + # Arrange + request = HttpRequest() + request.query_params = QueryDict("symbol=AAPL") + expected_queryset = FinancialDataModel.objects.filter(symbol="AAPL") + + # Act + view = FinancialDataAPIView() + actual_queryset = view.get_queryset(request) + + # Assert + self.maxDiff = None + self.assertEqual( + actual_queryset.query.__str__(), expected_queryset.query.__str__() + ) + + def test_get_queryset_symbol_start_date(self): + # Arrange + request = HttpRequest() + request.query_params = {"symbol": "AAPL", "start_date": "2022-01-01"} + expected_queryset = FinancialDataModel.objects.filter( + symbol="AAPL", date__gte=datetime.strptime("2022-01-01", "%Y-%m-%d") + ) + + # Act + view = FinancialDataAPIView() + actual_queryset = view.get_queryset(request) + + # Assert + self.maxDiff = None + self.assertEqual( + actual_queryset.query.__str__(), expected_queryset.query.__str__() + ) + + def test_get_queryset(self): + # Arrange + request = HttpRequest() + request.query_params = { + "symbol": "AAPL", + "start_date": "2022-01-01", + "end_date": "2023-01-01", + } + expected_queryset = FinancialDataModel.objects.filter( + symbol="AAPL", + date__gte=datetime.strptime("2022-01-01", "%Y-%m-%d"), + date__lte=datetime.strptime("2023-01-01", "%Y-%m-%d"), + ) + + # Act + view = FinancialDataAPIView() + actual_queryset = view.get_queryset(request) + + # Assert + self.maxDiff = None + self.assertEqual( + actual_queryset.query.__str__(), expected_queryset.query.__str__() + ) + + def test_get_queryset_no_data(self): + # Arrange + request = HttpRequest() + request.query_params = {} + expected_queryset = FinancialDataModel.objects.filter() + + # Act + view = FinancialDataAPIView() + actual_queryset = view.get_queryset(request) + + # Assert + self.maxDiff = None + self.assertEqual( + actual_queryset.query.__str__(), expected_queryset.query.__str__() + ) + + def test_get_queryset_invalid_data(self): + # Arrange + request = HttpRequest() + request.query_params = {"invalid_key": "test message"} + expected_queryset = FinancialDataModel.objects.filter() + + # Act + view = FinancialDataAPIView() + actual_queryset = view.get_queryset(request) + + # Assert + self.maxDiff = None + self.assertEqual( + actual_queryset.query.__str__(), expected_queryset.query.__str__() + ) + + def test_get_without_filters(self): + # Arrange + request = HttpRequest() + request.query_params = {} + request.method = "GET" + expected_data = { + "data": [], + "pagination": {"count": 0, "page": 1, "limit": 5, "pages": 1}, + "info": {"error": ""}, + } + + # Mock the queryset returned by get_queryset() + with mock.patch.object( + FinancialDataAPIView, "get_queryset" + ) as mock_get_queryset: + mock_get_queryset.return_value = self.queryset + + # Act + view = FinancialDataAPIView() + response = view.get(request) + + # Assert + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data, expected_data) + + def test_get_invalid_start_date(self): + # Arrange + request = HttpRequest() + request.query_params = {"symbol": "AAPL", "start_date": "invalid_date"} + request.method = "GET" + expected_data = { + "data": [], + "pagination": {}, + "info": {"error": "start_date must be in the format YYYY-MM-DD"}, + } + + view = FinancialDataAPIView() + + # Act + response = view.get(request) + + # Assert + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data, expected_data) + + def test_get_invalid_end_date(self): + # Arrange + request = HttpRequest() + request.query_params = {"symbol": "AAPL", "end_date": "invalid_date"} + request.method = "GET" + expected_data = { + "data": [], + "pagination": {}, + "info": {"error": "end_date must be in the format YYYY-MM-DD"}, + } + + view = FinancialDataAPIView() + + # Acts + response = view.get(request) + + # Assert + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data, expected_data) + + def test_get_when_queryset_has_no_results(self): + # Arrange + serializer_mock = Mock(spec=FinancialDataSerializer) + serializer_mock.data = [] + paginator_mock = Mock() + paginator_mock.page_size = 0 + paginator_mock.page.paginator.num_pages = 0 + paginator_mock.page.number = 1 + paginator_mock.paginate_queryset.return_value = [] + request = self.factory.get("/api/financial_data") + request.query_params = QueryDict() + view = FinancialDataAPIView() + view.get_queryset = Mock(return_value=FinancialDataModel.objects.none()) + view.pagination_class = Mock(return_value=paginator_mock) + view.serializer_class = Mock(return_value=serializer_mock) + + # Act + response = view.get(request) + + # Assert + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + response.data, + { + "data": [], + "pagination": {"count": 0, "page": 1, "limit": 5, "pages": 0}, + "info": {"error": ""}, + }, + ) + + def test_get_returns_data_when_queryset_has_results(self): + # Arrange + queryset = Mock(spec=FinancialDataModel.objects.all()) + queryset.count.return_value = 2 + serializer_data = [{"foo": "bar"}, {"baz": "qux"}] + serializer = Mock(spec=FinancialDataSerializer) + serializer.data = serializer_data + serializer.return_value = serializer + paginator = Mock() + paginator.page_size = 5 + paginator.paginate_queryset.return_value = serializer_data + paginator.page.paginator.num_pages = 1 + paginator.page.number = 1 + request = self.factory.get("/api/financial_data", {"limit": 5}) + request.query_params = QueryDict() + view = FinancialDataAPIView() + view.get_queryset = Mock(return_value=queryset) + view.serializer_class = Mock(return_value=serializer) + view.pagination_class = Mock(return_value=paginator) + + # Act + response = view.get(request) + + # Assert + view.get_queryset.assert_called_once_with(request) + view.serializer_class.assert_called_once_with(serializer_data, many=True) + view.pagination_class.assert_called_once_with() + paginator.paginate_queryset.assert_called_once_with(queryset, request) + self.assertEqual( + response.data, + { + "data": serializer_data, + "pagination": {"count": 2, "page": 1, "limit": 5, "pages": 1}, + "info": {"error": ""}, + }, + ) + self.assertEqual(response.status_code, 200) + + @patch("core.views.FinancialDataAPIView.get_queryset") + def test_get(self, queryset_mock): + queryset_mock.return_value = FinancialDataModel.objects.filter(symbol="AAPL") + response = self.client.get(self.url, {"symbol": "AAPL"}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["data"][0]["symbol"], "AAPL") + self.assertEqual(response.data["pagination"]["count"], 1) + self.assertEqual(response.data["pagination"]["page"], 1) + self.assertEqual(response.data["pagination"]["limit"], 5) + self.assertEqual(response.data["pagination"]["pages"], 1) + + @patch("core.views.FinancialDataAPIView.get_queryset") + def test_get_invalid_symbol(self, queryset_mock): + symbol = "invalid_symbol" + queryset_mock.return_value = FinancialDataModel.objects.filter(symbol=symbol) + response = self.client.get(self.url, {"symbol": symbol}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["data"]), 0) + self.assertEqual(response.data["pagination"]["count"], 0) + self.assertEqual(response.data["pagination"]["page"], 1) + self.assertEqual(response.data["pagination"]["limit"], 5) + self.assertEqual(response.data["pagination"]["pages"], 1) diff --git a/financial/core/urls.py b/financial/core/urls.py new file mode 100644 index 00000000..89a6f7f5 --- /dev/null +++ b/financial/core/urls.py @@ -0,0 +1,13 @@ +from django.urls import path +from . import views + +urlpatterns = [ + path(route="financial_data/", view=views.FinancialDataAPIView.as_view()), + path( + route="financial_data", + view=views.FinancialDataAPIView.as_view(), + name="financial_data", + ), + path(route="statistics/", view=views.StatisticsAPIView.as_view()), + path(route="statistics", view=views.StatisticsAPIView.as_view(), name="statistics"), +] diff --git a/financial/core/views.py b/financial/core/views.py new file mode 100644 index 00000000..2cf7f987 --- /dev/null +++ b/financial/core/views.py @@ -0,0 +1,141 @@ +from rest_framework.views import APIView +from rest_framework import status +from rest_framework.response import Response +from rest_framework.pagination import PageNumberPagination + +from django.db.models import Avg, Sum +from datetime import datetime + +from .models import FinancialDataModel +from .serializers import FinancialDataSerializer + + +class FinancialDataAPIView(APIView): + serializer_class = FinancialDataSerializer + pagination_class = PageNumberPagination + + def get(self, request, *args, **kwargs): + try: + # Get the queryset based on the filters provided in the request + queryset = self.get_queryset(request) + + # Paginate the results + paginator = self.pagination_class() + paginator.page_size = request.query_params.get("limit", 5) + result_page = paginator.paginate_queryset(queryset, request) + serializer = self.serializer_class(result_page, many=True) + + # Construct the response data + response_data = { + "data": serializer.data, + "pagination": { + "count": queryset.count(), + "page": paginator.page.number, + "limit": int(paginator.page_size), + "pages": paginator.page.paginator.num_pages, + }, + "info": {"error": ""}, + } + + return Response(response_data) + + except Exception as e: + # Return an error response if an exception is raised + return Response( + {"data": [], "pagination": {}, "info": {"error": str(e)}}, + status=status.HTTP_400_BAD_REQUEST, + ) + + def get_queryset(self, request): + # Get all FinancialDataModel objects + queryset = FinancialDataModel.objects.all() + + # Filter by start date if start_date parameter is provided in the request + start_date = request.query_params.get("start_date") + if start_date: + try: + start_date = datetime.strptime(start_date, "%Y-%m-%d") + except ValueError: + raise ValueError("start_date must be in the format YYYY-MM-DD") + queryset = queryset.filter(date__gte=start_date) + + # Filter by end date if end_date parameter is provided in the request + end_date = request.query_params.get("end_date") + if end_date: + try: + end_date = datetime.strptime(end_date, "%Y-%m-%d") + except ValueError: + raise ValueError("end_date must be in the format YYYY-MM-DD") + queryset = queryset.filter(date__lte=end_date) + + # Filter by symbol if symbol parameter is provided in the request + symbol = request.query_params.get("symbol") + if symbol: + queryset = queryset.filter(symbol=symbol) + + return queryset + + +class StatisticsAPIView(APIView): + serializer_class = FinancialDataModel + + def get(self, request, *args, **kwargs): + try: + # Get the required parameters from the query params + start_date = request.query_params.get("start_date") + end_date = request.query_params.get("end_date") + symbol = request.query_params.get("symbol") + + # Check if all required parameters are present + if not all([start_date, end_date, symbol]): + raise ValueError( + "start_date, end_date, and symbol are required parameters" + ) + + # Check if the start_date and end_date parameters are in the correct format + try: + start_date = datetime.strptime(start_date, "%Y-%m-%d") + end_date = datetime.strptime(end_date, "%Y-%m-%d") + except ValueError: + raise ValueError( + "start_date and end_date must be in the format YYYY-MM-DD" + ) + + # Check if the start_date parameter is before the end_date parameter + if start_date > end_date: + raise ValueError("start_date must be before end_date") + + # Get the queryset filtered by the required parameters + queryset = FinancialDataModel.objects.filter( + date__gte=start_date, date__lte=end_date, symbol=symbol + ) + + # Calculate the statistics + average_daily_open_price = queryset.aggregate(Avg("open_price"))[ + "open_price__avg" + ] + average_daily_close_price = queryset.aggregate(Avg("close_price"))[ + "close_price__avg" + ] + average_daily_volume = queryset.aggregate(Sum("volume"))["volume__sum"] + + # Construct the response data + response_data = { + "data": { + "start_date": start_date, + "end_date": end_date, + "symbol": symbol, + "average_daily_open_price": average_daily_open_price, + "average_daily_close_price": average_daily_close_price, + "average_daily_volume": average_daily_volume, + }, + "info": {"error": ""}, + } + + return Response(response_data) + + except Exception as e: + return Response( + {"data": {}, "info": {"error": str(e)}}, + status=status.HTTP_400_BAD_REQUEST, + ) diff --git a/financial/financial/asgi.py b/financial/financial/asgi.py new file mode 100644 index 00000000..c58c21a1 --- /dev/null +++ b/financial/financial/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for financial project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "financial.settings") + +application = get_asgi_application() diff --git a/financial/financial/settings.py b/financial/financial/settings.py new file mode 100644 index 00000000..b1f64cfa --- /dev/null +++ b/financial/financial/settings.py @@ -0,0 +1,121 @@ +""" +Django settings for financial project. + +Generated by 'django-admin startproject' using Django 4.1.7. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.1/ref/settings/ +""" + +import os +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-6fpnw#5!-36*r#*z#+iub@en95tt=9x#3z4vcvx*k$*u&#k$=n" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + # financial data apps + "rest_framework", + "core", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "financial.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ] + }, + } +] + +WSGI_APPLICATION = "financial.wsgi.application" + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": os.getenv("DB_NAME", "financial"), + "USER": os.getenv("DB_USER", "ritheesh"), + "PASSWORD": os.getenv("DB_PASSWORD", "ritheeshPassword1"), + "HOST": os.getenv("DB_HOST", "127.0.0.1"), + "PORT": os.getenv("DB_PORT", "3306"), + }, + "test": {"ENGINE": "django.db.backends.sqlite3"}, +} + +# Password validation +# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" + }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.1/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.1/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/financial/financial/test_settings.py b/financial/financial/test_settings.py new file mode 100644 index 00000000..bfeebc16 --- /dev/null +++ b/financial/financial/test_settings.py @@ -0,0 +1,5 @@ +from .settings import * + +DATABASES = { + "default": {"ENGINE": "django.db.backends.sqlite3", "NAME": BASE_DIR / "db.sqlite3"} +} diff --git a/financial/financial/urls.py b/financial/financial/urls.py new file mode 100644 index 00000000..4efb8721 --- /dev/null +++ b/financial/financial/urls.py @@ -0,0 +1,24 @@ +"""financial URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.1/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path, include +from . import views + +urlpatterns = [ + path("admin/", admin.site.urls), + path("api/", include("core.urls")), + path("", views.health_check, name="health_check"), +] diff --git a/financial/financial/views.py b/financial/financial/views.py new file mode 100644 index 00000000..3a0caf14 --- /dev/null +++ b/financial/financial/views.py @@ -0,0 +1,5 @@ +from django.http import HttpResponse + + +def health_check(request): + return HttpResponse("Service is up and running") diff --git a/financial/financial/wsgi.py b/financial/financial/wsgi.py new file mode 100644 index 00000000..048c396e --- /dev/null +++ b/financial/financial/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for financial project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "financial.settings") + +application = get_wsgi_application() diff --git a/financial/manage.py b/financial/manage.py new file mode 100644 index 00000000..271c767c --- /dev/null +++ b/financial/manage.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + test_database = os.getenv("TEST_DATABASE", "OFF") + if test_database == "ON": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "financial.test_settings") + else: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "financial.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/get_raw_data.py b/get_raw_data.py new file mode 100644 index 00000000..c5feec23 --- /dev/null +++ b/get_raw_data.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +import os +import requests +import logging +import mysql.connector +from mysql.connector import errorcode +from datetime import datetime, timedelta +from pathlib import Path +from dotenv import load_dotenv + +load_dotenv() + +BASE_DIR = Path(__file__).resolve().parent +SCHEMA_FILE = BASE_DIR / "schema.sql" + +API_KEY = os.getenv("ALPHAVANTAGE_API_KEY", "15SWOEC7H3CLW3B1") +SYMBOLS = ["IBM", "AAPL"] +DB_NAME = os.getenv("MYSQL_DATABASE", "financial") +USER = os.getenv("MYSQL_USER", "ritheesh") +PASSWORD = os.getenv("MYSQL_PASSWORD", "ritheeshPassword1") +HOST = os.getenv("HOST", "127.0.0.1") + +# Set up logging +logging.basicConfig( + filename="get_raw_data.log", + filemode="w", + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(message)s", +) + + +def get_financial_data(symbol): + """ + Retrieve financial data for a given stock symbol from AlphaVantage API for the past two weeks. + + :param symbol: str, stock symbol to retrieve data for + :return: list of dict, each dict contains financial data for a single day + """ + try: + url = f"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={symbol}&apikey={API_KEY}" + response = requests.get(url) + response.raise_for_status() + data = response.json()["Time Series (Daily)"] + today = datetime.now().date() + two_weeks_ago = today - timedelta(days=14) + records = [] + for date, values in data.items(): + date = datetime.strptime(date, "%Y-%m-%d").date() + if date >= two_weeks_ago and date <= today: + record = { + "symbol": symbol, + "date": date.isoformat(), + "open_price": values["1. open"], + "close_price": values["4. close"], + "volume": values["6. volume"], + } + records.append(record) + return records + except requests.exceptions.RequestException as e: + logging.error( + f"Failed to retrieve financial data for symbol {symbol}: {str(e)}" + ) + raise e + except (ValueError, KeyError) as e: + logging.error(f"Unexpected response format for symbol {symbol}: {str(e)}") + raise e + + +def create_financial_data_table(conn): + """ + Create a new table named 'financial_data' in the database with the specified connection. + + :param conn: mysql.connector.connection_cext.CMySQLConnection object, connection to the database + :return: None + """ + try: + cursor = conn.cursor() + cursor.execute(f"CREATE DATABASE IF NOT EXISTS {DB_NAME}") + cursor.execute(f"USE {DB_NAME}") + with open(SCHEMA_FILE, "r") as f: + schema = f.read() + cursor.execute(schema, multi=True) + conn.commit() + logging.info("Created financial_data table") + except mysql.connector.Error as e: + logging.error(f"Failed to create financial_data table: {str(e)}") + raise e + except Exception as e: + logging.error(f"Unknown error creating financial_data table: {e}") + raise e + + +def insert_financial_data(conn, records, table_name="financial_data"): + """ + Insert financial data records into the 'financial_data' table in the database with the specified connection. + + :param conn: mysql.connector.connection_cext.CMySQLConnection object, connection to the database + :param records: list of dict, each dict contains financial data for a single day + :return: None + """ + try: + cursor = conn.cursor() + for record in records: + cursor.execute( + """ + INSERT INTO {} (symbol, date, open_price, close_price, volume) + VALUES (%s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE open_price=VALUES(open_price), close_price=VALUES(close_price), volume=VALUES(volume) + """.format( + table_name + ), + ( + record["symbol"], + record["date"], + record["open_price"], + record["close_price"], + record["volume"], + ), + ) + conn.commit() + logging.info( + f"Inserted {len(records)} financial data records for symbol {records[0]['symbol']}" + ) + for record in records: + logging.debug(record) + except mysql.connector.Error as e: + logging.error(f"Error inserting financial data into database: {str(e)}") + conn.rollback() + except Exception as e: + logging.error(f"Unknown error inserting financial_data table: {e}") + raise e + + +def main(): + """ + Main function that retrieves financial data for the specified stock symbols and inserts them into the database. + + :return: None + """ + conn = None + try: + conn = mysql.connector.connect(user=USER, password=PASSWORD, host=HOST) + create_financial_data_table(conn) + for symbol in SYMBOLS: + logging.info(f"Retrieving financial data for symbol {symbol}") + records = get_financial_data(symbol) + insert_financial_data(conn, records) + except mysql.connector.Error as e: + if e.errno == errorcode.ER_ACCESS_DENIED_ERROR: + logging.error("Something is wrong with your user name or password") + elif e.errno == errorcode.ER_BAD_DB_ERROR: + logging.error("Database does not exist") + else: + logging.error(f"Error connecting to database: {str(e)}") + raise e + finally: + if conn: + conn.close() + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..8fd7db61 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +Django>=4.1.7,<5.0 +djangorestframework>=3.14.0,<4.0 +requests>=2.28.2,<3.0 +mysql-connector-python>=8.0.32,<9.0 +mysqlclient>=2.1.1,<3.0 +python-dotenv diff --git a/schema.sql b/schema.sql new file mode 100644 index 00000000..be71ba1b --- /dev/null +++ b/schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS financial_data ( + id INT AUTO_INCREMENT PRIMARY KEY, + symbol VARCHAR(255) NOT NULL, + date DATE NOT NULL, + open_price DECIMAL(10,2) NOT NULL, + close_price DECIMAL(10,2) NOT NULL, + volume BIGINT NOT NULL +); diff --git a/tests/test_get_raw_data.py b/tests/test_get_raw_data.py new file mode 100644 index 00000000..7b4ef9f7 --- /dev/null +++ b/tests/test_get_raw_data.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +import unittest +import mysql.connector + +from unittest import mock +from unittest.mock import patch, MagicMock +from pathlib import Path +from get_raw_data import ( + get_financial_data, + create_financial_data_table, + insert_financial_data, + main, + SYMBOLS, +) + +BASE_DIR = Path(__file__).resolve().parent +DB_NAME = "financial" +SCHEMA_FILE = BASE_DIR / "schema.sql" +SYMBOL = "AAPL" + + +class TestGetFinancialData(unittest.TestCase): + @patch("requests.get") + def test_positive(self, mock_get): + # Arrange + # simulate a successful API response + mock_response = { + "Time Series (Daily)": { + "2023-03-10": { + "1. open": "123.45", + "4. close": "124.56", + "6. volume": "7890123", + }, + "2023-03-09": { + "1. open": "122.34", + "4. close": "123.45", + "6. volume": "6789012", + }, + } + } + mock_get.return_value.json.return_value = mock_response + mock_get.return_value.raise_for_status.return_value = None + + # Act + result = get_financial_data("AAPL") + + # Assert + # check that the result is as expected + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["symbol"], "AAPL") + self.assertEqual(result[0]["open_price"], "123.45") + self.assertEqual(result[0]["close_price"], "124.56") + self.assertEqual(result[0]["volume"], "7890123") + self.assertEqual(result[1]["symbol"], "AAPL") + self.assertEqual(result[1]["open_price"], "122.34") + self.assertEqual(result[1]["close_price"], "123.45") + self.assertEqual(result[1]["volume"], "6789012") + + @patch("requests.get") + def test_api_error(self, mock_get): + # Arrange + # simulate an API error + mock_get.return_value.raise_for_status.side_effect = Exception("API error") + + # Act & Assert + # call the function and check that it raises an exception + with self.assertRaises(Exception): + get_financial_data("AAPL") + + @patch("requests.get") + def test_response_format_error(self, mock_get): + # Arrange + # simulate an unexpected API response format + mock_response = {"unexpected key": "unexpected value"} + mock_get.return_value.json.return_value = mock_response + mock_get.return_value.raise_for_status.return_value = None + + # Act & Assert + # call the function and check that it raises an exception + with self.assertRaises(Exception): + get_financial_data("AAPL") + + +class TestCreateFinancialDataTable2(unittest.TestCase): + @patch("mysql.connector") + def test_create_financial_data_table_positive(self, mock_connector): + # Arrange + # Setup mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_connector.connect.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + + # Act + create_financial_data_table(mock_conn) + + # Assert + self.assertEqual(mock_cursor.execute.call_count, 3) + mock_cursor.execute.assert_any_call(f"CREATE DATABASE IF NOT EXISTS {DB_NAME}") + mock_cursor.execute.assert_any_call(f"USE {DB_NAME}") + mock_cursor.execute.assert_any_call(mock.ANY, multi=True) + mock_conn.commit.assert_called_once() + + @patch("mysql.connector") + def test_create_financial_data_table_negative(self, mock_connector): + # Arrange + # Setup mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_connector.connect.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + + # Raise an exception when execute is called on cursor + mock_cursor.execute.side_effect = Exception("Test exception") + + # Act + # Call function and expect it to raise an exception + with self.assertRaises(Exception): + create_financial_data_table(mock_conn) + + # Assert + # Assert that necessary database calls were made + self.assertEqual(mock_cursor.execute.call_count, 1) + mock_cursor.execute.assert_any_call(f"CREATE DATABASE IF NOT EXISTS {DB_NAME}") + + +class TestInsertFinancialData(unittest.TestCase): + @patch("mysql.connector") + def test_insert_successful(self, mock_connector): + # Arrange + # Setup mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_connector.connect.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + + # Define test input + records = [ + { + "symbol": "AAPL", + "date": "2022-03-10", + "open_price": 200.0, + "close_price": 205.0, + "volume": 1000000, + }, + { + "symbol": "AAPL", + "date": "2022-03-09", + "open_price": 198.0, + "close_price": 202.0, + "volume": 900000, + }, + ] + + # Act + insert_financial_data(mock_conn, records) + + # Assert + self.assertEqual(mock_cursor.execute.call_count, len(records)) + self.assertTrue(mock_conn.cursor.called) + self.assertEqual(mock_conn.commit.call_count, 1) + + @patch("mysql.connector") + def test_insert_failed(self, mock_connector): + # Arrange + # Setup mock connection and cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_connector.connect.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + + mock_cursor.execute.side_effect = Exception("Test exception") + + records = [ + { + "symbol": "AAPL", + "date": "2022-03-10", + "open_price": 174.80, + "close_price": 175.95, + "volume": 30843130, + }, + { + "symbol": "AAPL", + "date": "2022-03-09", + "open_price": 173.60, + "close_price": 174.14, + "volume": 26804000, + }, + ] + + # Act + with self.assertRaises(Exception): + insert_financial_data(mock_conn, records) + + # Assert + self.assertTrue(mock_conn.cursor.called) + self.assertFalse(mock_conn.commit.called) + + @patch.object(mysql.connector, "connect") + def test_insert_sql_error(self, mock_connect): + # Arrange + mock_cursor = MagicMock() + mock_cursor.execute.side_effect = mysql.connector.Error() + mock_connect.return_value.cursor.return_value = mock_cursor + mock_conn = mock_connect.return_value + records = [ + { + "symbol": "AAPL", + "date": "2022-03-10", + "open_price": 100.0, + "close_price": 101.0, + "volume": 1000000, + } + ] + + # Act + insert_financial_data(mock_conn, records) + + # Assert + mock_conn.rollback.assert_called_once() + + +class TestMain(unittest.TestCase): + @patch("mysql.connector.connect") + @patch("get_raw_data.create_financial_data_table") + @patch("get_raw_data.get_financial_data") + @patch("get_raw_data.insert_financial_data") + def test_main( + self, + mock_insert_financial_data, + mock_get_financial_data, + mock_create_financial_data_table, + mock_mysql_connector_connect, + ): + # Arrange + mock_cursor = MagicMock() + mock_conn = MagicMock() + mock_mysql_connector_connect.return_value = mock_conn + mock_conn.cursor.return_value = mock_cursor + + # Mock get_financial_data to return a list of two records + mock_get_financial_data.return_value = [ + { + "symbol": SYMBOL, + "date": "2023-03-11", + "open_price": 100, + "close_price": 110, + "volume": 10000, + }, + { + "symbol": SYMBOL, + "date": "2023-03-12", + "open_price": 110, + "close_price": 120, + "volume": 15000, + }, + ] + + # Act + main() + + # Assert + # Check that create_financial_data_table was called with the correct argument + mock_create_financial_data_table.assert_called_once_with(mock_conn) + + # Check that get_financial_data was called once for each symbol + mock_get_financial_data.assert_has_calls( + [mock.call(symbol) for symbol in SYMBOLS] + ) + + # Check that insert_financial_data was called twice with the correct arguments + mock_insert_financial_data.assert_has_calls( + mock_conn, + [ + MagicMock(records=[mock_get_financial_data.return_value[0]]), + MagicMock(records=[mock_get_financial_data.return_value[1]]), + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_process_raw_data.py b/tests/test_process_raw_data.py new file mode 100644 index 00000000..b4b5c80f --- /dev/null +++ b/tests/test_process_raw_data.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +import unittest +import mysql.connector +import datetime + +from decimal import Decimal +from unittest.mock import patch, mock_open +from pathlib import Path +from get_raw_data import ( + get_financial_data, + create_financial_data_table, + insert_financial_data, +) + +SYMBOL = "AAPL" +BASE_DIR = Path(__file__).resolve().parent +SCHEMA_FILE = BASE_DIR / "tests" / "test_schema.sql" +DB_NAME = "financial" +TABLE_NAME = "test_financial_data" + +USER = "ritheesh" +PASSWORD = "Mysql123" +HOST = "127.0.0.1" + + +class TestGetRawData(unittest.TestCase): + def setUp(self): + self.conn = mysql.connector.connect(user=USER, password=PASSWORD, host=HOST) + self.cursor = self.conn.cursor() + self.cursor.execute(f"USE {DB_NAME}") + + def tearDown(self): + self.cursor.execute(f"DROP TABLE IF EXISTS {TABLE_NAME}") + self.conn.close() + + def test_get_financial_data(self): + # Arrange & Act + data = get_financial_data(SYMBOL) + + # Assert + self.assertTrue(isinstance(data, list)) + if len(data) > 0: + self.assertTrue(isinstance(data[0], dict)) + self.assertTrue("symbol" in data[0]) + self.assertTrue("date" in data[0]) + self.assertTrue("open_price" in data[0]) + self.assertTrue("close_price" in data[0]) + self.assertTrue("volume" in data[0]) + + def test_get_financial_data_invalid_symbol(self): + # Arrange, Act, Assert + with self.assertRaises(KeyError): + get_financial_data("INVALID_SYMBOL") + + def test_create_financial_data_table(self): + # Arrange + mock_schema_contents = """ + CREATE TABLE IF NOT EXISTS test_financial_data ( + id INT AUTO_INCREMENT PRIMARY KEY, + symbol VARCHAR(255) NOT NULL, + date DATE NOT NULL, + open_price DECIMAL(10,2) NOT NULL, + close_price DECIMAL(10,2) NOT NULL, + volume BIGINT NOT NULL + ); + """ + with patch( + "builtins.open", mock_open(read_data=mock_schema_contents) + ) as mock_file: + # Act + create_financial_data_table(self.conn) + + # Assert + cursor = self.conn.cursor() + cursor.execute(f"SHOW TABLES LIKE '{TABLE_NAME}'") + table_exists = cursor.fetchone() + self.assertIsNotNone(table_exists) + + def test_insert_financial_data(self): + # Arrange + records = [ + { + "symbol": SYMBOL, + "date": "2023-03-10", + "open_price": "220.92", + "close_price": "222.46", + "volume": "29474759", + }, + { + "symbol": SYMBOL, + "date": "2023-03-09", + "open_price": "217.67", + "close_price": "218.89", + "volume": "39344709", + }, + ] + + mock_schema_contents = """ + CREATE TABLE IF NOT EXISTS test_financial_data ( + id INT AUTO_INCREMENT PRIMARY KEY, + symbol VARCHAR(255) NOT NULL, + date DATE NOT NULL, + open_price DECIMAL(10,2) NOT NULL, + close_price DECIMAL(10,2) NOT NULL, + volume BIGINT NOT NULL + ); + """ + + with patch( + "builtins.open", mock_open(read_data=mock_schema_contents) + ) as mock_file: + # Act + create_financial_data_table(self.conn) + + # Assert + insert_financial_data(self.conn, records, TABLE_NAME) + cursor = self.conn.cursor() + cursor.execute(f"SELECT * FROM {TABLE_NAME} WHERE symbol='AAPL'") + inserted_records = cursor.fetchall() + self.assertEqual(len(inserted_records), 2) + + expected_record_1 = ( + 1, + "AAPL", + datetime.date(2023, 3, 10), + Decimal("220.92").quantize(Decimal(".01")), + Decimal("222.46").quantize(Decimal(".01")), + 29474759, + ) + expected_record_2 = ( + 2, + SYMBOL, + datetime.date(2023, 3, 9), + Decimal("217.67").quantize(Decimal(".01")), + Decimal("218.89").quantize(Decimal(".01")), + 39344709, + ) + self.assertEqual(inserted_records[0], expected_record_1) + self.assertEqual(inserted_records[1], expected_record_2) diff --git a/tests/test_schema.sql b/tests/test_schema.sql new file mode 100644 index 00000000..dc9f4e12 --- /dev/null +++ b/tests/test_schema.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS test_financial_data ( + id INT AUTO_INCREMENT PRIMARY KEY, + symbol VARCHAR(255) NOT NULL, + date DATE NOT NULL, + open_price DECIMAL(10,2) NOT NULL, + close_price DECIMAL(10,2) NOT NULL, + volume BIGINT NOT NULL +);