diff --git a/.all-contributorsrc b/.all-contributorsrc new file mode 100644 index 0000000..db3c7ec --- /dev/null +++ b/.all-contributorsrc @@ -0,0 +1,190 @@ +{ + "projectName": "python", + "projectOwner": "logdna", + "repoType": "github", + "repoHost": "https://github.com", + "files": [ + "README.md" + ], + "imageSize": 100, + "commit": true, + "commitConvention": "angular", + "contributors": [ + { + "login": "respectus", + "name": "Muaz Siddiqui", + "avatar_url": "https://avatars.githubusercontent.com/u/1046364?v=4", + "profile": "https://github.com/respectus", + "contributions": [ + "code", + "doc", + "test" + ] + }, + { + "login": "smusali", + "name": "Samir Musali", + "avatar_url": "https://avatars.githubusercontent.com/u/34287490?v=4", + "profile": "https://github.com/smusali", + "contributions": [ + "code", + "doc", + "test" + ] + }, + { + "login": "vilyapilya", + "name": "vilyapilya", + "avatar_url": "https://avatars.githubusercontent.com/u/17367511?v=4", + "profile": "https://github.com/vilyapilya", + "contributions": [ + "code", + "maintenance", + "test" + ] + }, + { + "login": "mikehu", + "name": "Mike Hu", + "avatar_url": "https://avatars.githubusercontent.com/u/981800?v=4", + "profile": "https://github.com/mikehu", + "contributions": [ + "doc" + ] + }, + { + "login": "esatterwhite", + "name": "Eric Satterwhite", + "avatar_url": "https://avatars.githubusercontent.com/u/148561?v=4", + "profile": "http://codedependant.net/", + "contributions": [ + "code", + "doc", + "test", + "tool" + ] + }, + { + "login": "utek", + "name": "Łukasz Bołdys (Lukasz Boldys)", + "avatar_url": "https://avatars.githubusercontent.com/u/128036?v=4", + "profile": "http://dev.utek.pl/", + "contributions": [ + "code", + "bug" + ] + }, + { + "login": "baronomasia", + "name": "Ryan", + "avatar_url": "https://avatars.githubusercontent.com/u/4133158?v=4", + "profile": "https://github.com/baronomasia", + "contributions": [ + "doc" + ] + }, + { + "login": "LYHuang", + "name": "Mike Huang", + "avatar_url": "https://avatars.githubusercontent.com/u/14082239?v=4", + "profile": "https://github.com/LYHuang", + "contributions": [ + "code", + "bug" + ] + }, + { + "login": "danmaas", + "name": "Dan Maas", + "avatar_url": "https://avatars.githubusercontent.com/u/9013104?v=4", + "profile": "https://www.medium.com/@dmaas", + "contributions": [ + "code" + ] + }, + { + "login": "dchai76", + "name": "DChai", + "avatar_url": "https://avatars.githubusercontent.com/u/13873467?v=4", + "profile": "https://github.com/dchai76", + "contributions": [ + "doc" + ] + }, + { + "login": "jdemaeyer", + "name": "Jakob de Maeyer ", + "avatar_url": "https://avatars.githubusercontent.com/u/10531844?v=4", + "profile": "https://naboa.de/", + "contributions": [ + "code" + ] + }, + { + "login": "sataloger", + "name": "Andrey Babak", + "avatar_url": "https://avatars.githubusercontent.com/u/359111?v=4", + "profile": "https://github.com/sataloger", + "contributions": [ + "code" + ] + }, + { + "login": "SpainTrain", + "name": "Mike S", + "avatar_url": "https://avatars.githubusercontent.com/u/380032?v=4", + "profile": "https://github.com/mike-spainhower", + "contributions": [ + "code", + "doc" + ] + }, + { + "login": "btashton", + "name": "Brennan Ashton", + "avatar_url": "https://avatars.githubusercontent.com/u/173245?v=4", + "profile": "https://github.com/btashton", + "contributions": [ + "code" + ] + }, + { + "login": "inkrement", + "name": "Christian Hotz-Behofsits", + "avatar_url": "https://avatars.githubusercontent.com/u/604528?v=4", + "profile": "http://justrocketscience.com/", + "contributions": [ + "code", + "bug" + ] + }, + { + "login": "kurtiss", + "name": "Kurtiss Hare", + "avatar_url": "https://avatars.githubusercontent.com/u/108118?v=4", + "profile": "http://www.kinoarts.com/blog/", + "contributions": [ + "bug" + ] + }, + { + "login": "rudyryk", + "name": "Alexey Kinev", + "avatar_url": "https://avatars.githubusercontent.com/u/4500?v=4", + "profile": "https://twitter.com/rudyryk", + "contributions": [ + "bug" + ] + }, + { + "login": "matthiasfru", + "name": "matthiasfru", + "avatar_url": "https://avatars.githubusercontent.com/u/24245643?v=4", + "profile": "https://github.com/matthiasfru", + "contributions": [ + "bug" + ] + } + ], + "contributorsPerLine": 7 +} diff --git a/.config.mk b/.config.mk new file mode 100644 index 0000000..65038c5 --- /dev/null +++ b/.config.mk @@ -0,0 +1,9 @@ +# Below is an example of pulling the current version of a node app. + +#VERSION is being deprecated by APP_VERSION - no changes necessary - see Makefile +#APP_VERSION=$(shell awk '/version/ {gsub(/[",]/,""); print $$2}' package.json) + +GIT_AUTHOR_NAME ?= $(shell git config --get user.name) +GIT_AUTHOR_EMAIL ?= $(shell git config --get user.email) +GIT_COMMITTER_NAME ?= $(GIT_AUTHOR_NAME) +GIT_COMMITTER_EMAIL ?= $(GIT_AUTHOR_EMAIL) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc6e372 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +*.pyc +**/*cache* +build/ +develop-eggs +dist +src/*.egg-info +logdna.egg-info/ +LogDNAHandler.log +bin +.installed.cfg +coverage/ +.coverage +Session.vim +.cache +pypoetry/ +pip/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b52389a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,127 @@ +# Changelog + + + +## v1.18.12 (2023-07-27) +### Fix +* Don't overwrite base logging Handler class lock var ([#108](https://github.com/logdna/python/issues/108)) ([`5b04d72`](https://github.com/logdna/python/commit/5b04d72b42686d926fb5e73dadb2d7a1ba16d9c3)) + +## v1.18.11 (2023-07-26) +### Fix +* Remove Thread/event to fix Django regression ([#107](https://github.com/logdna/python/issues/107)) ([`0337a09`](https://github.com/logdna/python/commit/0337a09433953227dabb0b65da8583e8f9273986)) + +## v1.18.10 (2023-07-26) +### Fix +* Utilize apikey header for auth to make compatible with pipelines ([#104](https://github.com/logdna/python/issues/104)) ([`5394e97`](https://github.com/logdna/python/commit/5394e9714779878cd415a4566cd44d9183e150b9)) + +## v1.18.9 (2023-07-21) +### Fix +* Make flush thread a daemon thread to prevent shutdown hang ([#102](https://github.com/logdna/python/issues/102)) ([`17a69b0`](https://github.com/logdna/python/commit/17a69b044de43a7a9d7d3e6eb65a0c60f1fa23f0)) + +## v1.18.8 (2023-07-18) +### Fix +* Bump semver ([#101](https://github.com/logdna/python/issues/101)) ([`913e5f4`](https://github.com/logdna/python/commit/913e5f4f35f2920a6b2162022b93495b4774654c)) + +## v1.18.7 (2023-05-05) +### Fix +* Gate buils from non maintainers ([`97803b5`](https://github.com/logdna/python/commit/97803b55a102539a75b0763891c1d27757460067)) + +## v1.18.6 (2023-01-27) +### Fix +* Added retries for http 429 504 ([#93](https://github.com/logdna/python/issues/93)) ([`712d81d`](https://github.com/logdna/python/commit/712d81d4ab2bfbf95d65898fb365a5bcb1396199)) + +## v1.18.5 (2023-01-27) +### Fix +* **chore:** Upgraded pytest to resolve security vulnerability in py <= 1.11.0 ([#92](https://github.com/logdna/python/issues/92)) ([`905b06a`](https://github.com/logdna/python/commit/905b06a648e19896087b0c51ba1e055212727560)) + +## v1.18.4 (2023-01-06) +### Fix +* Dependabot -> Vulnerabilities -> cryptography >= 37.0.0 < 38.0.3 ([#91](https://github.com/logdna/python/issues/91)) ([`7ccde50`](https://github.com/logdna/python/commit/7ccde50ba50c0abbec1a7efe4dd665e8b35511c0)) + +## v1.18.3 (2022-12-07) +### Fix +* Add documentation for log_error_response option ([#88](https://github.com/logdna/python/issues/88)) ([`d5bf85c`](https://github.com/logdna/python/commit/d5bf85ca26579e0186e1abdbcd1b6c44c60c9eca)) + +## v1.18.2 (2022-05-10) +### Fix +* Requests error handling ([#82](https://github.com/logdna/python/issues/82)) ([`e412859`](https://github.com/logdna/python/commit/e4128592aa9c2301c6467115148f2f23f88da9d3)) + +## v1.18.1 (2021-11-18) +### Fix +* **threading:** Account for secondary buffer flush and deadlock ([`d00b952`](https://github.com/logdna/python/commit/d00b9529d116ddf9ba454462d4d804dc54423c83)) + +## v1.18.0 (2021-07-26) +### Fix +* **opts:** Repair logging options for each call ([`e326e4c`](https://github.com/logdna/python/commit/e326e4c2461b808b5d3a885b37555f8e610615e4)) + +## v1.17.0 (2021-07-15) +### Feature +* **meta:** Enable adding custom meta fields ([`f250237`](https://github.com/logdna/python/commit/f250237dbde932e99ab199023516f66a248c5e80)) +* **threadWorkerPools:** Introduce extra threads ([`9bfe479`](https://github.com/logdna/python/commit/9bfe479132acb0aa8e5784f2aa31298606e49789)) + +### Documentation +* Update the README as requested ([`5dd754f`](https://github.com/logdna/python/commit/5dd754f177675eb25cd5d5449bd3bcc8286f8739)) + +## v1.16.0 (2021-04-15) + + +## v1.15.0 (2021-04-15) + + +## v1.14.0 (2021-04-15) + + +## v1.13.0 (2021-04-15) + + +## v1.12.0 (2021-04-15) + + +## v1.11.0 (2021-04-15) + + +## v1.10.0 (2021-04-15) + + +## v1.9.0 (2021-04-15) + + +## v1.8.0 (2021-04-15) + + +## v1.7.0 (2021-04-07) + + +## v1.6.0 (2021-04-07) +### Feature +* **ci:** Enable releases through semantic release ([`9e4b1c0`](https://github.com/logdna/python/commit/9e4b1c0a43bc0941ba4fb336ea12a3f497622ee6)) +* **ci:** Include jenkins setup ([`1c5be37`](https://github.com/logdna/python/commit/1c5be37ef32776f2d0ba2b68b67cebb58b1f0177)) +* **tasks:** Convert run scripts to tasks ([`f5a8b18`](https://github.com/logdna/python/commit/f5a8b182941a0c514594cbaa7a3ac5952173d5a8)) +* **lint:** Setup linting + test harness ([`0a7763a`](https://github.com/logdna/python/commit/0a7763a2befbf598b59d7ab595b19e22b173fda5)) +* **tags:** Allow tags to be configured ([`6adc21e`](https://github.com/logdna/python/commit/6adc21e872fa521be1aaf08309f7f3d0ba3dc5c5)) +* **meta:** Python log info in meta object ([`cf9b505`](https://github.com/logdna/python/commit/cf9b505734df12918a665a8a8c74d4fd74e5bc47)) +* **handlers:** Make available via config file ([`39c5dec`](https://github.com/logdna/python/commit/39c5decd98e8d4feb6c1bbfa487faf35396c8b12)) + +### Fix +* **ci:** Correct invalid environment variable ([`fce4d59`](https://github.com/logdna/python/commit/fce4d5995b31c426f2b66992b57f129f22f9f18f)) + +### Documentation +* Add @matthiasfru as a contributor ([`3727bc3`](https://github.com/logdna/python/commit/3727bc3386d3dd6465e0b0d15675a091a3743c24)) +* Add @rudyryk as a contributor ([`9ab0e39`](https://github.com/logdna/python/commit/9ab0e3932180c41bff1cd0944c24fb8e208f4391)) +* Add @kurtiss as a contributor ([`1476085`](https://github.com/logdna/python/commit/14760857649b56207240bae907386a33f7f1666b)) +* Update @inkrement as a contributor ([`6a2cede`](https://github.com/logdna/python/commit/6a2cedef6695e18a981a3605176c9cffdd159827)) +* Add @inkrement as a contributor ([`7ba6992`](https://github.com/logdna/python/commit/7ba6992287f98c478da72f5982a0869b5d835df7)) +* Add @btashton as a contributor ([`c1e69bf`](https://github.com/logdna/python/commit/c1e69bfc965e1e1e5717ec7af3271dc0bf6b502d)) +* Add @SpainTrain as a contributor ([`7b9f04b`](https://github.com/logdna/python/commit/7b9f04b86a1e813bccea29ea8db1554808ea48b6)) +* Add @sataloger as a contributor ([`04081f0`](https://github.com/logdna/python/commit/04081f039d6136a66b259f7245ad7845ef1c080a)) +* Add @jdemaeyer as a contributor ([`69e8f7c`](https://github.com/logdna/python/commit/69e8f7cc0782a778d0da230ff1a8feb5f8cee352)) +* Add @dchai76 as a contributor ([`42e9c6b`](https://github.com/logdna/python/commit/42e9c6bddc2bce29714072bba37d85d9a734eac2)) +* Add @danmaas as a contributor ([`23bbab4`](https://github.com/logdna/python/commit/23bbab48cdef4aaef6813459dbdb23dbd9c60374)) +* Add @LYHuang as a contributor ([`3cc9e23`](https://github.com/logdna/python/commit/3cc9e232d0b12b9f0315efb1d618426b486a2604)) +* Add @baronomasia as a contributor ([`8971bd7`](https://github.com/logdna/python/commit/8971bd713d74cd9386c60a50cc75a7fc3591f544)) +* Add @utek as a contributor ([`ea495b4`](https://github.com/logdna/python/commit/ea495b479cc0549ff68a1e816c3f7a174c1c138c)) +* Add @esatterwhite as a contributor ([`0a334bd`](https://github.com/logdna/python/commit/0a334bdb5690049634c64eb0f9c6c1026b9ab001)) +* Add @mikehu as a contributor ([`e7aa7cb`](https://github.com/logdna/python/commit/e7aa7cb2624313c065f7bbc8a129c1a4841f9ec2)) +* Add @vilyapilya as a contributor ([`e3191f5`](https://github.com/logdna/python/commit/e3191f577fb7ddf7590ca1e1bf239f66d2f30fd0)) +* Add @smusali as a contributor ([`4d5022f`](https://github.com/logdna/python/commit/4d5022f93948cca239ebc104e34034c350956f65)) +* Add @respectus as a contributor ([`85a543c`](https://github.com/logdna/python/commit/85a543c9dc27a3c6064e790be0ba1c475187c38d)) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..0d5c4b2 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,138 @@ +# Contributor's Code of Conduct + +If you contribute to this repo, you agree to abide by this code of conduct for +this community. + +We abide by the +[Contributor Covenant Code of Conduct, 2.0](https://www.contributor-covenant.org/version/2/0/code_of_conduct/). +It is reproduced below for ease of use. + +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[opensource@logdna.com](mailto:opensource@logdna.com). +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2032386 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,37 @@ +# Contributing + +## Process + +We use a fork-and-PR process, also known as a triangular workflow. This process +is fairly common in open-source projects. Here's the basic workflow: + +1. Fork the upstream repo to create your own repo. This repo is called the origin repo. +2. Clone the origin repo to create a working directory on your local machine. +3. Work your changes on a branch in your working directory, then add, commit, and push your work to your origin repo. +4. Submit your changes as a PR against the upstream repo. You can use the upstream repo UI to do this. +5. Maintainers review your changes. If they ask for changes, you work on your + origin repo's branch and then submit another PR. Otherwise, if no changes are made, + then the branch with your PR is merged to upstream's main trunk, the master branch. + +When you work in a triangular workflow, you have the upstream repo, the origin +repo, and then your working directory (the clone of the origin repo). You do +a `git fetch` from upstream to local, push from local to origin, and then do a PR from origin to +upstream—a triangle. + +If this workflow is too much to understand to start, that's ok! You can use +GitHub's UI to make a change, which is autoset to do most of this process for +you. We just want you to be aware of how the entire process works before +proposing a change. + +Thank you for your contributions; we appreciate you! + +## License + +Note that we use a standard [MIT](./LICENSE) license on this repo. + +## Coding style + +Currently the project is auto formatted following the [PEP8][] +style guide + +[PEP8]: https://www.python.org/dev/peps/pep-0008 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..db887bb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM condaforge/miniforge3:4.12.0-0 + +RUN conda install -y gcc pip poetry=1.1.7 git +RUN mkdir /workdir && chmod 777 /workdir +RUN git config --global --add safe.directory /workdir +WORKDIR /workdir + diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..a516452 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,102 @@ +library 'magic-butler-catalogue' +def PROJECT_NAME = 'logdna-python' +def TRIGGER_PATTERN = ".*@logdnabot.*" +def DEFAULT_BRANCH = 'master' +def CURRENT_BRANCH = [env.CHANGE_BRANCH, env.BRANCH_NAME]?.find{branch -> branch != null} + +pipeline { + agent { + node { + label 'ec2-fleet' + customWorkspace "${PROJECT_NAME}-${BUILD_NUMBER}" + } + } + + options { + timestamps() + ansiColor 'xterm' + } + + triggers { + issueCommentTrigger(TRIGGER_PATTERN) + } + + stages { + stage('Validate PR Source') { + when { + expression { env.CHANGE_FORK } + not { + triggeredBy 'issueCommentCause' + } + } + steps { + error("A maintainer needs to approve this PR for CI by commenting") + } + } + stage('Test') { + + steps { + sh 'make install lint test' + } + + post { + always { + junit 'coverage/test.xml' + publishHTML target: [ + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'coverage', + reportFiles: 'index.html', + reportName: "coverage-${BUILD_TAG}" + ] + } + } + } + + stage('Release') { + + stages { + stage('dry run') { + when { + not { + branch "${DEFAULT_BRANCH}" + } + } + + environment { + GH_TOKEN = credentials('github-api-token') + PYPI_TOKEN = credentials('pypi-token') + JENKINS_URL = "${JENKINS_URL}" + BRANCH_NAME = "${DEFAULT_BRANCH}" + GIT_BRANCH = "${DEFAULT_BRANCH}" + CHANGE_ID = '' + } + + steps { + sh "make release-dry" + } + } + + stage('publish') { + + environment { + GH_TOKEN = credentials('github-api-token') + PYPI_TOKEN = credentials('pypi-token') + JENKINS_URL = "${JENKINS_URL}" + } + + when { + branch "${DEFAULT_BRANCH}" + not { + changelog '\\[skip ci\\]' + } + } + steps { + sh 'make release' + } + } + } + } + } +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a7f8886 --- /dev/null +++ b/Makefile @@ -0,0 +1,101 @@ +# Makefile Version 2021032403 +# +# Source in repository specific environment variables +include .config.mk + +# Define commands via docker +DOCKER = docker +DOCKER_RUN := $(DOCKER) run --rm -i +WORKDIR :=/workdir +DOCKER_COMMAND := $(DOCKER_RUN) -u "$(shell id -u)":"$(shell id -g)" -v $(PWD):$(WORKDIR):Z -w $(WORKDIR) \ + -e XDG_CONFIG_HOME=$(WORKDIR) \ + -e XDG_CACHE_HOME=$(WORKDIR) \ + -e POETRY_CACHE_DIR=$(WORKDIR)/.cache \ + -e POETRY_VIRTUALENV_IN_PROJECT=true \ + -e PYPI_TOKEN \ + -e GH_TOKEN \ + -e JENKINS_URL \ + -e BRANCH_NAME \ + -e CHANGE_ID \ + -e GIT_AUTHOR_NAME \ + -e GIT_AUTHOR_EMAIL \ + -e GIT_COMMITTER_NAME \ + -e GIT_COMMITTER_EMAIL \ + logdna-poetry:local + + +POETRY_COMMAND := $(DOCKER_COMMAND) poetry + +# Exports the variables for shell use +export + +# build image +.PHONY:build-image +build-image: + DOCKER_BUILDKIT=1 $(DOCKER) build -t logdna-poetry:local . + +# This helper function makes debugging much easier. +.PHONY:debug-% +debug-%: ## Debug a variable by calling `make debug-VARIABLE` + @echo $(*) = $($(*)) + +.PHONY:help +.SILENT:help +help: ## Show this help, includes list of all actions. + @awk 'BEGIN {FS = ":.*?## "}; /^.+: .*?## / && !/awk/ {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' ${MAKEFILE_LIST} + +.PHONY:run +run: install ## purge build time artifacts + $(DOCKER_COMMAND) bash + +.PHONY:clean +clean: ## purge build time artifacts + rm -rf dist/ build/ coverage/ pypoetry/ pip/ **/__pycache__/ .pytest_cache/ .cache .coverage + +.PHONY:changelog +changelog: install ## print the next version of the change log to stdout + $(POETRY_COMMAND) run semantic-release changelog --unreleased + +.PHONY:install +install: build-image ## install development and build time dependencies + $(POETRY_COMMAND) install --no-interaction + +.PHONY:lint +lint: install ## run lint rules and print error report + $(POETRY_COMMAND) run task lint + +.PHONY:lint-fix +lint-fix: install ## attempt to auto fix linting error and report remaining errors + $(POETRY_COMMAND) run task lint:fix + +.PHONY:package +package: install ## Generate a python sdist and wheel + $(POETRY_COMMAND) build + +.PHONY:release +release: clean install fetch-tags ## run semantic release build and publish results to github + pypi based on unreleased commits + $(POETRY_COMMAND) run task release + +.PHONY: fetch-tags +fetch-tags: ## workaround for jenkins repo cloning behavior + git fetch origin --tags + +.PHONY:release-dry +release-dry: clean install fetch-tags changelog ## run semantic release in noop mode + $(POETRY_COMMAND) run semantic-release publish --noop --verbosity=DEBUG + +.PHONY:release-patch +release-patch: clean install ## run semantic release build and force a patch release + $(POETRY_COMMAND) run semantic-release publish --patch + +.PHONY:release-minor +release-minor: clean install ## run semantic release build and force a minor release + $(POETRY_COMMAND) run semantic-release publish --minor + +.PHONY:release-major +release-major: clean install ## run semantic release build and force a major release + $(POETRY_COMMAND) run semantic-release publish --major + +.PHONY:test +test: install ## run project test suite + $(POETRY_COMMAND) run task test diff --git a/README.md b/README.md index e0b4354..8db2bf3 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,28 @@

Python package for logging to LogDNA

---- - -* **[Install](#install)** -* **[Setup](#setup)** -* **[Usage](#usage)** -* **[API](#api)** -* **[License](#license)** + +[![All Contributors](https://img.shields.io/badge/all_contributors-18-orange.svg?style=flat-square)](#contributors-) + +--- -## Install +* [Installation](#installation) +* [Setup](#setup) +* [Usage](#usage) + * [Usage with fileConfig](#usage-with-fileconfig) +* [API](#api) + * [LogDNAHandler(key: string, [options: dict])](#logdnahandlerkey-string-options-dict) + * [key](#key) + * [options](#options) + * [log(line, [options])](#logline-options) + * [line](#line) + * [options](#options-1) +* [Development](#development) + * [Scripts](#scripts) +* [License](#license) + +## Installation ```bash $ pip install logdna @@ -24,24 +36,32 @@ $ pip install logdna ```python import logging from logdna import LogDNAHandler +import os -key = 'YOUR INGESTION KEY HERE' +# Set your key as an env variable +# then import here, its best not to +# hard code your key! +key=os.environ['INGESTION_KEY'] log = logging.getLogger('logdna') log.setLevel(logging.INFO) options = { - 'hostname': 'pytest' + 'hostname': 'pytest', + 'ip': '10.0.1.1', + 'mac': 'C0:FF:EE:C0:FF:EE' } -# Defaults to false, when true ensures meta object will be searchable +# Defaults to False; when True meta objects are searchable options['index_meta'] = True +options['custom_fields'] = 'meta' + test = LogDNAHandler(key, options) log.addHandler(test) -log.warn("Warning message", {'app': 'bloop'}) +log.warning("Warning message", extra={'app': 'bloop'}) log.info("Info message") ``` @@ -49,11 +69,10 @@ _**Required**_ * [LogDNA Ingestion Key](https://app.logdna.com/manage/profile) _**Optional**_ -* Hostname - *(String)* - max length 32 chars -* MAC Address - *(String)* -* IP Address - *(String)* -* Max Length - *(Boolean)* - formatted as options['max_length'] -* Index Meta - *(Boolean)* - formatted as options['index_meta'] +* Hostname - ([string][]) +* MAC Address - ([string][]) +* IP Address - ([string][]) +* Index Meta - ([bool][]) - formatted as `options['index_meta']` ## Usage @@ -63,35 +82,70 @@ After initial setup, logging is as easy as: log.info('My Sample Log Line') # Add a custom level -log.info('My Sample Log Line', { 'level': 'MyCustomLevel' }) +log.info('My Sample Log Line', extra={ 'level': 'MyCustomLevel' }) # Include an App name with this specific log -log.info('My Sample Log Line', { 'level': 'Warn', 'app': 'myAppName'}) +log.info('My Sample Log Line', extra={ 'level': 'Warn', 'app': 'myAppName'}) -# Pass any associated objects along as metadata +# Pass associated objects along as metadata meta = { 'foo': 'bar', 'nested': { - 'nest1': 'nested text' + 'nest1': 'nested text' } } opts = { - 'level': 'warn', - 'meta': meta + 'level': 'warn', + 'meta': meta } -log.info('My Sample Log Line', opts) +log.info('My Sample Log Line', extra=opts) +``` + +### Usage with fileConfig + +To use [LogDNAHandler](#logdnahandlerkey-string-options-dict) with [fileConfig][] (e.g., in a Django `settings.py` file): + +```python +import os +import logging +from logdna import LogDNAHandler # required to register `logging.handlers.LogDNAHandler` + +LOGGING = { + # Other logging settings... + 'handlers': { + 'logdna': { + 'level': logging.DEBUG, + 'class': 'logging.handlers.LogDNAHandler', + 'key': os.environ.get('LOGDNA_INGESTION_KEY'), + 'options': { + 'app': '', + 'env': os.environ.get('ENVIRONMENT'), + 'index_meta': , + }, + }, + }, + 'loggers': { + '': { + 'handlers': ['logdna'], + 'level': logging.DEBUG + }, + }, +} ``` + +(This example assumes you have set environment variables for `ENVIRONMENT` and `LOGDNA_INGESTION_KEY`.) + ## API -### LogDNAHandler(key, [options]) +### LogDNAHandler(key: [string][], [options: [dict][]]) #### key -_**Required**_ -Type: `String` -Values: `YourAPIKey` +* _**Required**_ +* Type: [string][] +* Values: `` The [LogDNA API Key](https://app.logdna.com/manage/profile) associated with your account. @@ -99,106 +153,170 @@ The [LogDNA API Key](https://app.logdna.com/manage/profile) associated with your ##### app -_Optional_ -Type: `String` -Default: `''` -Values: `YourCustomApp` -Max Length: `32` +* _Optional_ +* Type: [string][] +* Default: `''` +* Values: `` + +The default app named that is included in every every log line sent through this instance. + +##### env -The default app passed along with every log sent through this instance. +* _Optional_ +* Type: [string][] +* Default: `''` +* Values: `` + +The default env passed along with every log sent through this instance. ##### hostname -_Optional_ -Type: `String` -Default: `''` -Values: `YourCustomHostname` -Max Length: `32` +* _Optional_ +* Type: [string][] +* Default: `''` +* Values: `` The default hostname passed along with every log sent through this instance. -##### index_meta +##### include_standard_meta -_Optional_ -Type: `Boolean` -Default: `False` +* _Optional_ +* Type: [bool][] +* Default: `False` -We allow meta objects to be passed with each line. By default these meta objects will be stringified and will not be searchable, but will be displayed for informational purposes. +Python [LogRecord][] objects includes language-specific information that may be useful metadata in logs. +Setting `include_standard_meta` to `True` automatically populates meta objects with `name`, `pathname`, and `lineno` +from the [LogRecord][]. -If this option is turned to true then meta objects will be parsed and will be searchable up to three levels deep. Any fields deeper than three levels will be stringified and cannot be searched. +*WARNING* This option is deprecated and will be removed in the upcoming major release. -*WARNING* When this option is true, your metadata objects across all types of log messages MUST have consistent types or the metadata object may not be parsed properly! +##### index_meta + +* _Optional_ +* Type: [bool][] +* Default: `False` + +We allow meta objects to be passed with each line. By default these meta objects are stringified and not searchable, and are only displayed for informational purposes. + +If this option is set to True then meta objects are parsed and searchable up to three levels deep. Any fields deeper than three levels are stringified and cannot be searched. +*WARNING* If this option is True, your metadata objects MUST have consistent types across all log messages or the metadata object might not be parsed properly. ##### level -_Optional_ -Type: `String` -Default: `Info` -Values: `Debug`, `Trace`, `Info`, `Warn`, `Error`, `Fatal`, `YourCustomLevel` -Max Length: `32` +* _Optional_ +* Type: [string][] +* Default: `Info` +* Values: `Debug`, `Trace`, `Info`, `Warn`, `Error`, `Fatal`, `` The default level passed along with every log sent through this instance. +##### verbose + +* _Optional_ +* Type: [string][] or [bool][] +* Default: `True` +* Values: `False` or any level + +Sets the verbosity of log statements for failures. + +##### request_timeout + +* _Optional_ +* Type: [int][] +* Default: `30000` + +The amount of time (in ms) the request should wait for LogDNA to respond before timing out. + +##### tags + +* _Optional_ +* Type: [list][]<[string][]> +* Default: `[]` -##### max_length +List of tags used to dynamically group hosts. More information on tags is available at [How Do I Use Host Tags?](https://docs.logdna.com/docs/logdna-agent#section-how-do-i-use-host-tags-) -_Optional_ -Type: `Boolean` -Default: `True` +##### url -By default the line has a maximum length of 16000 chars, this can be turned off with the value false. +* _Optional_ +* Type: [string][] +* Default: `'https://logs.logdna.com/logs/ingest'` +A custom ingestion endpoint to stream log lines into. + +##### custom_fields + +* _Optional_ +* Type: [list][]<[string][]> +* Default: `['args', 'name', 'pathname', 'lineno']` + +List of fields out of `record` object to include in the `meta` object. By default, `args`, `name`, `pathname`, and `lineno` will be included. + +##### log_error_response + +* _Optional_ +* Type: [bool][] +* Default: `False` + +Enables logging of the API response when an HTTP error is encountered ### log(line, [options]) #### line -_Required_ -Type: `String` -Default: `''` -Max Length: `32000` +* _Required_ +* Type: [string][] +* Default: `''` -The line which will be sent to the LogDNA system. +The log line to be sent to LogDNA. #### options ##### level -_Optional_ -Type: `String` -Default: `Info` -Values: `Debug`, `Trace`, `Info`, `Warn`, `Error`, `Fatal`, `YourCustomLevel` -Max Length: `32` +* _Optional_ +* Type: [string][] +* Default: `Info` +* Values: `Debug`, `Trace`, `Info`, `Warn`, `Error`, `Fatal`, `` The level passed along with this log line. ##### app -_Optional_ -Type: `String` -Default: `''` -Values: `YourCustomApp` -Max Length: `32` +* _Optional_ +* Type: [string][] +* Default: `''` +* Values: `` The app passed along with this log line. +##### env + +* _Optional_ +* Type: [string][] +* Default: `''` +* Values: `` + +The environment passed with this log line. + ##### meta -_Optional_ -Type: `JSON` -Default: `None` +* _Optional_ +* Type: [dict][] +* Default: `None` + +A standard dictonary containing additional metadata about the log line that is passed. Please ensure values are JSON serializable. -A meta object for additional metadata about the log line that is passed. +**NOTE**: Values that are not JSON serializable will be removed and the respective keys will be added to the `__errors` string. ##### index_meta -_Optional_ -Type: `Boolean` -Default: `False` +* _Optional_ +* Type: [bool][] +* Default: `False` -We allow meta objects to be passed with each line. By default these meta objects will be stringified and will not be searchable, -but will be displayed for informational purposes. +We allow meta objects to be passed with each line. By default these meta objects will be stringified and will not be +searchable, but will be displayed for informational purposes. If this option is turned to true then meta objects will be parsed and will be searchable up to three levels deep. Any fields deeper than three levels will be stringified and cannot be searched. @@ -206,14 +324,105 @@ If this option is turned to true then meta objects will be parsed and will be se ##### timestamp -_Optional_ -Default: `time.time()` +* _Optional_ +* Type: [float][] +* Default: [time.time()][] + +The time in seconds since the epoch to use for the log timestamp. It must be within one day or current time - if it is not, it is ignored and time.time() is used in its place. + + +## Development + +This project makes use of the [poetry][] package manager for local development. + +```shell +$ poetry install +``` -A timestamp in ms, must be within one day otherwise it will be dropped and time.time() will be used in its place. +### Scripts +**lint** +Run linting rules w/o attempting to fix them + +```shell +$ poetry run task lint +``` + + +**lint:fix** + +Run lint rules against all local python files and attempt to fix where possible. + + +```shell +$ poetry run task lint:fix +``` + +**test**: + +Runs all unit tests and generates coverage reports + +```shell +poetry run task test +``` + +## Contributors ✨ + +Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Muaz Siddiqui
💻 📖 ⚠️
Samir Musali
💻 📖 ⚠️
vilyapilya
💻 🚧 ⚠️
Mike Hu
📖
Eric Satterwhite
💻 📖 ⚠️ 🔧
Łukasz Bołdys (Lukasz Boldys)
💻 🐛
Ryan
📖

Mike Huang
💻 🐛
Dan Maas
💻
DChai
📖
Jakob de Maeyer
💻
Andrey Babak
💻
Mike S
💻 📖
Brennan Ashton
💻

Christian Hotz-Behofsits
💻 🐛
Kurtiss Hare
🐛
Alexey Kinev
🐛
matthiasfru
🐛
+ + + + + + +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! ## License MIT © [LogDNA](https://logdna.com/) +Copyright © 2017 [LogDNA][], released under an MIT license. See the [LICENSE](./LICENSE) file and [https://opensource.org/licenses/MIT](https://opensource.org/licenses/MIT) + *Happy Logging!* + +[bool]: https://docs.python.org/3/library/stdtypes.html#boolean-values +[dict]: https://docs.python.org/3/library/stdtypes.html#mapping-types-dict +[int]: https://docs.python.org/3/library/functions.html#int +[float]: https://docs.python.org/3/library/functions.html#float +[string]: https://docs.python.org/3/library/string.html +[list]: https://docs.python.org/3/library/stdtypes.html#list +[time.time()]: https://docs.python.org/3/library/time.html?#time.time +[poetry]: https://python-poetry.org +[LogDNA]: https://logdna.com/ +[LogRecord]: https://docs.python.org/2/library/logging.html#logrecord-objects +[fileConfig]: https://docs.python.org/2/library/logging.config.html#logging.config.fileConfig diff --git a/logdna/VERSION b/logdna/VERSION new file mode 100644 index 0000000..43bae12 --- /dev/null +++ b/logdna/VERSION @@ -0,0 +1 @@ +1.18.12 diff --git a/logdna/__init__.py b/logdna/__init__.py index e617e37..27abc1a 100644 --- a/logdna/__init__.py +++ b/logdna/__init__.py @@ -1,3 +1,8 @@ from .logdna import LogDNAHandler __all__ = ['LogDNAHandler'] +# Publish this class to the "logging.handlers" module so that it can be use +# from a logging config file via logging.config.fileConfig(). +import logging.handlers + +logging.handlers.LogDNAHandler = LogDNAHandler diff --git a/logdna/configs.py b/logdna/configs.py index aa42218..d790776 100644 --- a/logdna/configs.py +++ b/logdna/configs.py @@ -1,11 +1,19 @@ +from os import path, sep + +with open("{p}{s}VERSION".format(p=path.abspath(path.dirname(__file__)), + s=sep)) as f: + version = f.read().strip('\n') + defaults = { - 'CONTENT_TYPE': 'application/json; charset=UTF-8', - 'DEFAULT_REQUEST_TIMEOUT': 180000, - 'MS_IN_A_DAY': 86400000, - 'MAX_REQUEST_TIMEOUT': 30, - 'MAX_LINE_LENGTH': 32000, - 'MAX_INPUT_LENGTH': 32, - 'FLUSH_INTERVAL': 5, - 'FLUSH_BYTE_LIMIT': 1000000, - 'LOGDNA_URL': 'https://logs.logdna.com/logs/ingest' + 'DEFAULT_REQUEST_TIMEOUT': 30, + 'FLUSH_INTERVAL_SECS': 0.25, + 'FLUSH_LIMIT': 2 * 1024 * 1024, + 'MAX_CONCURRENT_REQUESTS': 10, + 'MAX_RETRY_ATTEMPTS': 3, + 'MAX_RETRY_JITTER': 0.5, + 'META_FIELDS': ['args', 'name', 'pathname', 'lineno'], + 'LOGDNA_URL': 'https://logs.logdna.com/logs/ingest', + 'BUF_RETENTION_LIMIT': 4 * 1024 * 1024, + 'RETRY_INTERVAL_SECS': 5, + 'USER_AGENT': 'python/%s' % version } diff --git a/logdna/logdna.py b/logdna/logdna.py index 70f864b..e9f6128 100644 --- a/logdna/logdna.py +++ b/logdna/logdna.py @@ -1,92 +1,355 @@ +import logging +import requests +import socket import sys +import threading import time -import json + +from concurrent.futures import ThreadPoolExecutor + from .configs import defaults -import logging -import requests +from .utils import sanitize_meta, get_ip, normalize_list_option -from threading import Timer -from socket import gethostname class LogDNAHandler(logging.Handler): - def __init__(self, token, options={}): - self.buf = [] + def __init__(self, key, options={}): + # Setup Handler logging.Handler.__init__(self) - self.token = token - self.hostname = options['hostname'] if 'hostname' in options else gethostname() - self.level = options['level'] if 'level' in options else 'info' - self.app = options['app'] if 'app' in options else '' + + # Set Internal Logger + self.internal_handler = logging.StreamHandler(sys.stdout) + self.internal_handler.setLevel(logging.DEBUG) + self.internalLogger = logging.getLogger('internal') + self.internalLogger.addHandler(self.internal_handler) + self.internalLogger.setLevel(logging.DEBUG) + + # Set the Custom Variables + self.key = key + self.hostname = options.get('hostname', socket.gethostname()) + self.ip = options.get('ip', get_ip()) + self.mac = options.get('mac', None) + self.loglevel = options.get('level', 'info') + self.app = options.get('app', '') + self.env = options.get('env', '') + self.tags = normalize_list_option(options, 'tags') + self.custom_fields = normalize_list_option(options, 'custom_fields') + self.custom_fields += defaults['META_FIELDS'] + self.log_error_response = options.get('log_error_response', False) + + # Set the Connection Variables + self.url = options.get('url', defaults['LOGDNA_URL']) + self.request_timeout = options.get('request_timeout', + defaults['DEFAULT_REQUEST_TIMEOUT']) + self.user_agent = options.get('user_agent', defaults['USER_AGENT']) + self.max_retry_attempts = options.get('max_retry_attempts', + defaults['MAX_RETRY_ATTEMPTS']) + self.max_retry_jitter = options.get('max_retry_jitter', + defaults['MAX_RETRY_JITTER']) + self.max_concurrent_requests = options.get( + 'max_concurrent_requests', defaults['MAX_CONCURRENT_REQUESTS']) + self.retry_interval_secs = options.get('retry_interval_secs', + defaults['RETRY_INTERVAL_SECS']) + + # Set the Flush-related Variables + self.buf = [] + self.buf_size = 0 + + self.include_standard_meta = options.get('include_standard_meta', None) + + if self.include_standard_meta is not None: + self.internalLogger.debug( + '"include_standard_meta" option will be deprecated ' + + 'removed in the upcoming major release') + + self.index_meta = options.get('index_meta', False) + self.flush_limit = options.get('flush_limit', defaults['FLUSH_LIMIT']) + self.flush_interval_secs = options.get('flush_interval', + defaults['FLUSH_INTERVAL_SECS']) + self.buf_retention_limit = options.get('buf_retention_limit', + defaults['BUF_RETENTION_LIMIT']) + + # Set up the Thread Pools + self.worker_thread_pool = ThreadPoolExecutor() + self.request_thread_pool = ThreadPoolExecutor( + max_workers=self.max_concurrent_requests) + self.setLevel(logging.DEBUG) + self._lock = threading.RLock() - self.max_length = True - if 'max_length' in options: - self.max_length = options['max_length'] - self.index_meta = False - if 'index_meta' in options: - self.index_meta = options['index_meta'] - self.flushLimit = defaults['FLUSH_BYTE_LIMIT'] - self.url = defaults['LOGDNA_URL'] - self.bufByteLength = 0 self.flusher = None - def bufferLog(self, message): - if message and message['line']: - if self.max_length and len(message['line']) > defaults['MAX_LINE_LENGTH']: - message['line'] = message['line'][:defaults['MAX_LINE_LENGTH']] + ' (cut off, too long...)' - print('Line was longer than ' + defaults['MAX_LINE_LENGTH'] + ' chars and was truncated.') + def start_flusher(self): + if not self.flusher: + self.flusher = threading.Timer( + self.flush_interval_secs, self.flush) + self.flusher.start() - self.bufByteLength += sys.getsizeof(message) - self.buf.append(message) + def close_flusher(self): + if self.flusher: + self.flusher.cancel() + self.flusher = None - if self.bufByteLength >= self.flushLimit: - self.flush() - return + def buffer_log(self, message): + if self.worker_thread_pool: + try: + self.worker_thread_pool.submit(self.buffer_log_sync, message) + except RuntimeError: + self.buffer_log_sync(message) + except Exception as e: + self.internalLogger.debug('Error in calling buffer_log: %s', e) - if not self.flusher: - self.flusher = Timer(defaults['FLUSH_INTERVAL'], self.flush) - self.flusher.start() + def buffer_log_sync(self, message): + # Attempt to acquire lock to write to buffer + if self._lock.acquire(blocking=True): + try: + msglen = len(message['line']) + if self.buf_size + msglen < self.buf_retention_limit: + self.buf.append(message) + self.buf_size += msglen + else: + self.internalLogger.debug( + 'The buffer size exceeded the limit: %s', + self.buf_retention_limit) + + if self.buf_size >= self.flush_limit: + self.close_flusher() + self.flush() + else: + self.start_flusher() + except Exception as e: + self.internalLogger.exception(f'Error in buffer_log_sync: {e}') + finally: + self._lock.release() def flush(self): - if not self.buf or len(self.buf) < 0: - return - data = {'e': 'ls', 'ls': self.buf} + self.schedule_flush_sync() + + def schedule_flush_sync(self, should_block=False): + if self.request_thread_pool: + try: + self.request_thread_pool.submit( + self.try_lock_and_do_flush_request, should_block) + except RuntimeError: + self.try_lock_and_do_flush_request(should_block) + except Exception as e: + self.internalLogger.debug( + 'Error in calling try_lock_and_do_flush_request: %s', e) + + def try_lock_and_do_flush_request(self, should_block=False): + local_buf = [] + if self._lock.acquire(blocking=should_block): + if not self.buf: + self.close_flusher() + self._lock.release() + return + + local_buf = self.buf.copy() + self.buf.clear() + self.buf_size = 0 + if local_buf: + self.close_flusher() + self._lock.release() + + if local_buf: + self.try_request(local_buf) + + def try_request(self, buf): + data = {'e': 'ls', 'ls': buf} + retries = 0 + while retries < self.max_retry_attempts: + retries += 1 + if self.send_request(data): + break + + sleep_time = self.retry_interval_secs * (1 << (retries - 1)) + sleep_time += self.max_retry_jitter + time.sleep(sleep_time) + + if retries >= self.max_retry_attempts: + self.internalLogger.debug( + 'Flush exceeded %s tries. Discarding flush buffer', + self.max_retry_attempts) + + def send_request(self, data): # noqa: max-complexity: 13 + """ + Send log data to LogDNA server + Returns: + True - discard flush buffer + False - retry, keep flush buffer + """ try: - resp = requests.post(url=defaults['LOGDNA_URL'], json=data, auth=('user', self.token), params={ 'hostname': self.hostname }, stream=True, timeout=defaults['MAX_REQUEST_TIMEOUT']) - self.buf = [] - self.bufByteLength = 0 - if self.flusher: - self.flusher.cancel() - self.flusher = None - except requests.exceptions.RequestException as e: - return + headers = { + 'user-agent': self.user_agent, + 'apikey': self.key + } + response = requests.post(url=self.url, + json=data, + params={ + 'hostname': self.hostname, + 'ip': self.ip, + 'mac': self.mac, + 'tags': self.tags, + 'now': int(time.time() * 1000) + }, + stream=True, + allow_redirects=True, + timeout=self.request_timeout, + headers=headers) + + status_code = response.status_code + ''' + response code: + 1XX unexpected status + 200 expected status, OK + 2XX unexpected status + 301 302 303 unexpected status, + per "allow_redirects=True" + 3XX unexpected status + 401, 403 expected client error, + invalid ingestion key + 429 expected server error, + "client error", transient + 4XX unexpected client error + 500 502 503 504 507 expected server error, transient + 5XX unexpected server error + handling: + expected status discard flush buffer + unexpected status log + discard flush buffer + expected client error log + discard flush buffer + unexpected client error log + discard flush buffer + expected server error log + retry + unexpected server error log + discard flush buffer + ''' + if status_code == 200: + return True # discard + + if isinstance(response.reason, bytes): + # We attempt to decode utf-8 first because some servers + # choose to localize their reason strings. If the string + # isn't utf-8, we fall back to iso-8859-1 for all other + # encodings. (See PR #3538) + try: + reason = response.reason.decode('utf-8') + except UnicodeDecodeError: + reason = response.reason.decode('iso-8859-1') + else: + reason = response.reason + + if 200 < status_code <= 399: + self.internalLogger.debug('Unexpected response: %s. ' + + 'Discarding flush buffer', + reason) + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return True # discard + + if status_code in [401, 403]: + self.internalLogger.debug( + 'Please provide a valid ingestion key. ' + + 'Discarding flush buffer') + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return True # discard + + if status_code == 429: + self.internalLogger.debug('Client Error: %s. Retrying...', + reason) + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return False # retry + + if 400 <= status_code <= 499: + self.internalLogger.debug('Client Error: %s. ' + + 'Discarding flush buffer', + reason) + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return True # discard + + if status_code in [500, 502, 503, 504, 507]: + self.internalLogger.debug('Server Error: %s. Retrying...', + reason) + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return False # retry + + self.internalLogger.debug('The request failed: %s.' + + 'Discarding flush buffer', + reason) + + except requests.exceptions.Timeout as timeout: + self.internalLogger.debug('Timeout Error: %s. Retrying...', + timeout) + return False # retry + + except requests.exceptions.RequestException as exception: + self.internalLogger.debug( + 'Error sending logs %s. Discarding flush buffer', exception) + + return True # discard def emit(self, record): + msg = self.format(record) record = record.__dict__ - opts = {} - if 'args' in record: - opts = record['args'] message = { - 'hostname' : self.hostname, - 'timestamp': int(time.time()), - 'line': record['msg'], - 'level': record['levelname'] or self.level, - 'app': self.app or record['module'] + 'hostname': self.hostname, + 'timestamp': int(time.time() * 1000), + 'line': msg, + 'level': record['levelname'] or self.loglevel, + 'app': self.app or record['module'], + 'env': self.env, + 'meta': {} } - if 'level' in opts: - message['level'] = opts['level'] - if 'app' in opts: - message['app'] = opts['app'] - if 'hostname' in opts: - message['hostname'] = opts['hostname'] - if 'timestamp' in opts: - message['timestamp'] = opts['timestamp'] - if 'meta' in opts: - if self.index_meta: - message['meta'] = opts['meta'] - else: - message['meta'] = json.dumps(opts['meta']) - self.bufferLog(message) + for key in self.custom_fields: + if key in record: + if isinstance(record[key], tuple): + message['meta'][key] = list(record[key]) + elif record[key] is not None: + message['meta'][key] = record[key] + + message['meta'] = sanitize_meta(message['meta'], self.index_meta) + + opts = {} + if 'args' in record and not isinstance(record['args'], tuple): + opts = record['args'] + + for key in ['app', 'env', 'hostname', 'level', 'timestamp']: + if key in opts: + message[key] = opts[key] + + self.buffer_log(message) def close(self): + # Close the flusher + self.close_flusher() + + # First gracefully shut down any threads that are still attempting + # to add log messages to the buffer. This ensures that we don't lose + # any log messages that are in the process of being added to the + # buffer. + if self.worker_thread_pool: + self.worker_thread_pool.shutdown(wait=True) + self.worker_thread_pool = None + + # Manually force a flush of any remaining log messages in the buffer. + # We block here to ensure that the flush completes prior to the + # application exiting and because the probability of this + # introducing a noticeable delay is very low because close() is only + # called when the logger and application are shutting down. + self.schedule_flush_sync(should_block=True) + + # Finally, shut down the thread pool that was used to send the log + # messages to the server. We can assume at this point that all log + # messages that were in the buffer prior to the worker threads + # shutting down have been sent to the server. + if self.request_thread_pool: + self.request_thread_pool.shutdown(wait=True) + self.request_thread_pool = None + logging.Handler.close(self) diff --git a/logdna/utils.py b/logdna/utils.py new file mode 100644 index 0000000..7278f12 --- /dev/null +++ b/logdna/utils.py @@ -0,0 +1,53 @@ +import json +import socket + + +def is_jsonable(obj): + try: + json.dumps(obj) + return True + except (TypeError, OverflowError, ValueError): + return False + + +def normalize_list_option(options, key): + value = options.get(key, []) + if isinstance(value, str): + value = [val.strip() for val in value.split(',')] + elif not isinstance(value, list): + value = [] + return value + + +def sanitize_meta(meta, index_meta=False): + if not index_meta: + if is_jsonable(meta): + return json.dumps(meta) + + return { + '__errors': 'Meta cannot be serialized into JSON-formatted string' + } + + keys_to_sanitize = [] + for key, value in meta.items(): + if not is_jsonable(value): + keys_to_sanitize.append(key) + if keys_to_sanitize: + for key in keys_to_sanitize: + del meta[key] + meta['__errors'] = 'These keys have been sanitized: ' + ', '.join( + keys_to_sanitize) + return meta + + +def get_ip(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + # doesn't even have to be reachable + s.connect(('10.255.255.255', 1)) + ip = s.getsockname()[0] + except Exception: + ip = '127.0.0.1' + finally: + s.close() + return ip diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..c7fb7c7 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,952 @@ +[[package]] +name = "appnope" +version = "0.1.3" +description = "Disable App Nap on macOS >= 10.9" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "attrs" +version = "22.2.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs"] +docs = ["furo", "sphinx", "myst-parser", "zope.interface", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] +tests = ["attrs", "zope.interface"] +tests-no-zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] +tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] + +[[package]] +name = "backcall" +version = "0.2.0" +description = "Specifications for callback functions passed in to an API" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "bleach" +version = "6.0.0" +description = "An easy safelist-based HTML-sanitizing tool." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +six = ">=1.9.0" +webencodings = "*" + +[package.extras] +css = ["tinycss2 (>=1.1.0,<1.2)"] + +[[package]] +name = "certifi" +version = "2022.12.7" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.0.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[[package]] +name = "click-log" +version = "0.4.0" +description = "Logging integration for Click" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +click = "*" + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" + +[[package]] +name = "coverage" +version = "5.5" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +toml = ["toml"] + +[[package]] +name = "cryptography" +version = "39.0.0" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1,!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +pep8test = ["black", "ruff"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] + +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +category = "dev" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "docutils" +version = "0.19" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "dotty-dict" +version = "1.3.1" +description = "Dictionary wrapper for quick access to deeply nested keys." +category = "dev" +optional = false +python-versions = ">=3.5,<4.0" + +[[package]] +name = "exceptiongroup" +version = "1.1.0" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "flake8" +version = "3.9.2" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.7.0,<2.8.0" +pyflakes = ">=2.3.0,<2.4.0" + +[[package]] +name = "gitdb" +version = "4.0.10" +description = "Git Object Database" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.30" +description = "GitPython is a python library used to interact with Git repositories" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +gitdb = ">=4.0.1,<5" +typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "importlib-metadata" +version = "6.0.0" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +perf = ["ipython"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"] + +[[package]] +name = "importlib-resources" +version = "5.10.2" +description = "Read resources from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "invoke" +version = "1.7.3" +description = "Pythonic task execution" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "ipdb" +version = "0.13.11" +description = "IPython-enabled pdb" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +decorator = {version = "*", markers = "python_version > \"3.6\" and python_version < \"3.11\" or python_version >= \"3.11\""} +ipython = {version = ">=7.31.1", markers = "python_version > \"3.6\" and python_version < \"3.11\" or python_version >= \"3.11\""} +tomli = {version = "*", markers = "python_version > \"3.6\" and python_version < \"3.11\""} + +[[package]] +name = "ipython" +version = "7.34.0" +description = "IPython: Productive Interactive Computing" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +appnope = {version = "*", markers = "sys_platform == \"darwin\""} +backcall = "*" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +pickleshare = "*" +prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0" +pygments = "*" +traitlets = ">=4.2" + +[package.extras] +all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.17)", "pygments", "qtconsole", "requests", "testpath"] +doc = ["Sphinx (>=1.3)"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["notebook", "ipywidgets"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.17)"] + +[[package]] +name = "jaraco.classes" +version = "3.2.3" +description = "Utility functions for Python class constructs" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +more-itertools = "*" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "jedi" +version = "0.18.2" +description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +parso = ">=0.8.0,<0.9.0" + +[package.extras] +docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx-rtd-theme (==0.4.3)", "sphinx (==1.8.5)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] + +[[package]] +name = "jeepney" +version = "0.8.0" +description = "Low-level, pure Python DBus protocol wrapper." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest", "pytest-trio", "pytest-asyncio (>=0.17)", "testpath", "trio", "async-timeout"] +trio = ["trio", "async-generator"] + +[[package]] +name = "keyring" +version = "23.13.1" +description = "Store and access your passwords safely." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +importlib-metadata = {version = ">=4.11.4", markers = "python_version < \"3.12\""} +importlib-resources = {version = "*", markers = "python_version < \"3.9\""} +"jaraco.classes" = "*" +jeepney = {version = ">=0.4.2", markers = "sys_platform == \"linux\""} +pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""} +SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} + +[package.extras] +completion = ["shtab"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] + +[[package]] +name = "matplotlib-inline" +version = "0.1.6" +description = "Inline Matplotlib backend for Jupyter" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +traitlets = "*" + +[[package]] +name = "mccabe" +version = "0.6.1" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "more-itertools" +version = "9.0.0" +description = "More routines for operating on iterables, beyond itertools" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "mslex" +version = "0.3.0" +description = "shlex for windows" +category = "dev" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "packaging" +version = "23.0" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "parso" +version = "0.8.3" +description = "A Python Parser" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["docopt", "pytest (<6.0.0)"] + +[[package]] +name = "pexpect" +version = "4.8.0" +description = "Pexpect allows easy control of interactive console applications." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +ptyprocess = ">=0.5" + +[[package]] +name = "pickleshare" +version = "0.7.5" +description = "Tiny 'shelve'-like database with concurrency support" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "pkginfo" +version = "1.9.6" +description = "Query metadata from sdists / bdists / installed packages." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +testing = ["pytest", "pytest-cov"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "prompt-toolkit" +version = "3.0.36" +description = "Library for building powerful interactive command lines in Python" +category = "dev" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +wcwidth = "*" + +[[package]] +name = "psutil" +version = "5.9.4" +description = "Cross-platform lib for process and system monitoring in Python." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +test = ["ipaddress", "mock", "enum34", "pywin32", "wmi"] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "pycodestyle" +version = "2.7.0" +description = "Python style guide checker" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pyflakes" +version = "2.3.1" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pygments" +version = "2.14.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "pytest" +version = "7.2.1" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "2.12.1" +description = "Pytest plugin for measuring coverage." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +coverage = ">=5.2.1" +pytest = ">=4.6" +toml = "*" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] + +[[package]] +name = "python-gitlab" +version = "3.12.0" +description = "Interact with GitLab API" +category = "dev" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +requests = ">=2.25.0" +requests-toolbelt = ">=0.9.1" + +[package.extras] +autocompletion = ["argcomplete (>=1.10.0,<3)"] +yaml = ["PyYaml (>=5.2)"] + +[[package]] +name = "python-semantic-release" +version = "7.33.0" +description = "Automatic Semantic Versioning for Python projects" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +click = ">=7,<9" +click-log = ">=0.3,<1" +dotty-dict = ">=1.3.0,<2" +gitpython = ">=3.0.8,<4" +invoke = ">=1.4.1,<2" +packaging = "*" +python-gitlab = ">=2,<4" +requests = ">=2.25,<3" +semver = ">=2.10,<3" +tomlkit = ">=0.10,<1.0" +twine = ">=3,<4" + +[package.extras] +dev = ["tox", "isort", "black"] +docs = ["Sphinx (==1.3.6)", "Jinja2 (==3.0.3)"] +mypy = ["mypy", "types-requests"] +test = ["coverage (>=5,<6)", "pytest (>=7,<8)", "pytest-xdist (>=1,<2)", "pytest-mock (>=2,<3)", "responses (==0.13.3)", "mock (==1.3.0)"] + +[[package]] +name = "pywin32-ctypes" +version = "0.2.0" +description = "" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "readme-renderer" +version = "37.3" +description = "readme_renderer is a library for rendering \"readme\" descriptions for Warehouse" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +bleach = ">=2.1.0" +docutils = ">=0.13.1" +Pygments = ">=2.5.1" + +[package.extras] +md = ["cmarkgfm (>=0.8.0)"] + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-toolbelt" +version = "0.10.1" +description = "A utility belt for advanced users of python-requests" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" + +[[package]] +name = "rfc3986" +version = "2.0.0" +description = "Validating URI References per RFC 3986" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +idna2008 = ["idna"] + +[[package]] +name = "secretstorage" +version = "3.3.3" +description = "Python bindings to FreeDesktop.org Secret Service API" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cryptography = ">=2.0" +jeepney = ">=0.6" + +[[package]] +name = "semver" +version = "2.13.0" +description = "Python helper for Semantic Versioning (http://semver.org/)" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "smmap" +version = "5.0.0" +description = "A pure Python implementation of a sliding window memory map manager" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "tap.py" +version = "3.1" +description = "Test Anything Protocol (TAP) tools" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +yaml = ["more-itertools", "PyYAML (>=5.1)"] + +[[package]] +name = "taskipy" +version = "1.10.3" +description = "tasks runner for python projects" +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +colorama = ">=0.4.4,<0.5.0" +mslex = {version = ">=0.3.0,<0.4.0", markers = "sys_platform == \"win32\""} +psutil = ">=5.7.2,<6.0.0" +tomli = {version = ">=2.0.1,<3.0.0", markers = "python_version >= \"3.7\" and python_version < \"4.0\""} + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "tomlkit" +version = "0.11.6" +description = "Style preserving TOML library" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "tqdm" +version = "4.64.1" +description = "Fast, Extensible Progress Meter" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "traitlets" +version = "5.8.1" +description = "Traitlets Python configuration system" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] +test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] + +[[package]] +name = "twine" +version = "3.8.0" +description = "Collection of utilities for publishing packages on PyPI" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = ">=0.4.3" +importlib-metadata = ">=3.6" +keyring = ">=15.1" +pkginfo = ">=1.8.1" +readme-renderer = ">=21.0" +requests = ">=2.20" +requests-toolbelt = ">=0.8.0,<0.9.0 || >0.9.0" +rfc3986 = ">=1.4.0" +tqdm = ">=4.14" +urllib3 = ">=1.26.0" + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "urllib3" +version = "1.26.14" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.extras] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "wcwidth" +version = "0.2.6" +description = "Measures the displayed width of unicode strings in a terminal" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "yapf" +version = "0.30.0" +description = "A formatter for Python code." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "zipp" +version = "3.11.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.7" +content-hash = "2ce3c41b1d2a1222b4e53674ab5901777d35e52bcee9eaeb93ff625c74e14d31" + +[metadata.files] +appnope = [] +attrs = [] +backcall = [] +bleach = [] +certifi = [] +cffi = [] +charset-normalizer = [] +click = [] +click-log = [] +colorama = [] +coverage = [] +cryptography = [] +decorator = [] +docutils = [] +dotty-dict = [] +exceptiongroup = [] +flake8 = [] +gitdb = [] +gitpython = [] +idna = [] +importlib-metadata = [] +importlib-resources = [] +iniconfig = [] +invoke = [] +ipdb = [] +ipython = [] +"jaraco.classes" = [] +jedi = [] +jeepney = [] +keyring = [] +matplotlib-inline = [] +mccabe = [] +more-itertools = [] +mslex = [] +packaging = [] +parso = [] +pexpect = [] +pickleshare = [] +pkginfo = [] +pluggy = [] +prompt-toolkit = [] +psutil = [] +ptyprocess = [] +pycodestyle = [] +pycparser = [] +pyflakes = [] +pygments = [] +pytest = [] +pytest-cov = [] +python-gitlab = [] +python-semantic-release = [] +pywin32-ctypes = [] +readme-renderer = [] +requests = [] +requests-toolbelt = [] +rfc3986 = [] +secretstorage = [] +semver = [] +six = [] +smmap = [] +"tap.py" = [] +taskipy = [] +toml = [] +tomli = [] +tomlkit = [] +tqdm = [] +traitlets = [] +twine = [] +typing-extensions = [] +urllib3 = [] +wcwidth = [] +webencodings = [] +yapf = [] +zipp = [] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..241a341 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +[tool.poetry] +name = "logdna" +version = "1.18.12" +description = 'A Python Package for Sending Logs to LogDNA' +authors = ["logdna "] +license = "MIT" + +[tool.semantic_release] +version_toml = "pyproject.toml:tool.poetry.version" +version_pattern = "logdna/VERSION:(\\d+\\.\\d+\\.\\d+)" +branch = "master" +commit_subject = "release: Version {version} [skip ci]" +commit_author = "LogDNA Bot " + +[tool.poetry.dependencies] +python = "^3.7" +requests = "^2.28.1" + +[tool.poetry.dev-dependencies] +coverage = "^5.4" +"tap.py" = "^3.0" +ipdb = "^0.13.4" +flake8 = "^3.8.4" +yapf = "^0.30.0" +pytest = "^7.2.0" +pytest-cov = "^2.11.1" +taskipy = "^1.6.0" +python-semantic-release = "^7.28.1" + +[tool.taskipy.tasks] +pre_test = "mkdir -p coverage" +test = "pytest --junitxml=coverage/test.xml --cov=logdna --cov-report=html --verbose tests/" +post_test = "python scripts/json_coverage.py" +lint = "flake8 --doctests" +"lint:fix" = "yapf -r -i logdna scripts tests" +"post_lint:fix" = "task lint" +release = "semantic-release publish" + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" + +[tool.pytest.ini_options] +minversion = "6.0" +testpaths = "tests" + +[tool.coverage.run] +branch = true +source = ["logdna"] + +[tool.coverage.report] +fail_under = 76 +show_missing = true + +[tool.coverage.json] +output = "coverage/coverage.json" + +[tool.coverage.html] +directory = "coverage" +show_contexts = true diff --git a/scripts/json_coverage.py b/scripts/json_coverage.py new file mode 100644 index 0000000..563d123 --- /dev/null +++ b/scripts/json_coverage.py @@ -0,0 +1,51 @@ +import json +from os import path +from coverage import Coverage + +ROOT = path.realpath(path.abspath(path.join(path.dirname(__file__), '..'))) +COVERAGE_DIR = path.join(ROOT, 'coverage') +JUNIT_PATH = path.join(COVERAGE_DIR, 'test.xml') + + +def json_coverage(): + COVERAGE_FILE = path.join(COVERAGE_DIR, 'coverage-final.json') + COVERAGE_SUMMARY = path.join(COVERAGE_DIR, 'coverage-summary.json') + + coverage = Coverage(config_file=path.join(ROOT, 'pyproject.toml')) + coverage.load() + coverage.json_report(outfile=COVERAGE_FILE) + + report = json.load(open(COVERAGE_FILE)) + totals = report.get('totals') + summary = { + 'lines': { + 'total': + totals['covered_lines'] + totals['missing_lines'], + 'covered': + totals['covered_lines'], + 'pct': + totals['covered_lines'] / + (totals['covered_lines'] + totals['missing_lines']) * 100 + }, + 'statements': { + 'total': None, + 'covered': None, + 'pct': None, + }, + 'functions': { + 'total': None, + 'covered': None, + 'pct': None, + }, + 'branches': { + 'total': totals['num_branches'], + 'covered': totals['covered_branches'], + 'pct': totals['covered_branches'] / totals['num_branches'] * 100 + } + } + + json.dump({'total': summary}, open(COVERAGE_SUMMARY, 'w')) + + +if __name__ == "__main__": + json_coverage() diff --git a/setup.cfg b/setup.cfg index b88034e..c24cd82 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,12 @@ [metadata] description-file = README.md + +[flake8] +exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,.cache,pip,pypoetry +max-complexity = 10 + +[yapf] +based_on_style = pep8 +indent_width = 4 +use_tabs = False + diff --git a/setup.py b/setup.py index 03908eb..fc16e48 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,39 @@ -from distutils.core import setup +from setuptools import setup +from os import path, sep + +# read the contents of your README file +this_directory = path.abspath(path.dirname(__file__)) + +with open(path.join(this_directory, 'README.md'), 'rb') as f: + long_description = f.read().decode('utf-8') + +kwargs = {"dir": this_directory, "sep": sep} +with open("{dir}{sep}logdna{sep}VERSION".format(**kwargs)) as f: + version = f.read().strip('\n') + setup( - name = 'logdna', - packages = ['logdna'], - version = '1.0.7', - description = 'A python package for sending logs to LogDNA', - author = 'Answerbook Inc.', - author_email = 'help@logdna.com', - url = 'https://github.com/logdna/python', - download_url = 'https://github.com/logdna/python/tarball/1.0.7', - keywords = ['logdna', 'logging', 'logs', 'python', 'logdna.com', 'logger'], - install_requires=[ - 'requests', - ], - classifiers = [], + name='logdna', + packages=['logdna'], + package_data={'': ['VERSION']}, + version=version, + description='A Python Package for Sending Logs to LogDNA', + author='LogDNA Inc.', + author_email='help@logdna.com', + license='MIT', + url='https://github.com/logdna/python', + download_url=('https://github.com/logdna/python/tarball/%s' % (version)), + keywords=['logdna', 'logging', 'logs', 'python', 'logdna.com', 'logger'], + install_requires=[ + 'requests', + ], + classifiers=[ + 'Topic :: System :: Logging', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9' + ], + long_description=long_description, + long_description_content_type='text/markdown', ) diff --git a/test.py b/test.py deleted file mode 100644 index 1a67b01..0000000 --- a/test.py +++ /dev/null @@ -1,32 +0,0 @@ -''' - An example of how to use the LogDNA Handler -''' - -import logging -import timeit -import sys - -from logdna import LogDNAHandler - -# from guppy import hpy -# h = hpy() - -key = 'YOUR API KEY HERE' -log = logging.getLogger('logdna') -log.setLevel(logging.INFO) -test = LogDNAHandler(key, { 'hostname': 'pytest' }) - -log.addHandler(test) - -log.warn("Warning message", {'app': 'bloop'}) -log.info("Info message") -# print h.heap() - -# Lines will be in order upon refresh -def timeThis(): - for x in range(100): - log.info('DINGLEBOP ' + str(x)) - -print (timeit.timeit(timeThis, number=2)) - -# print h.heap() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..cff2fe6 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +from tap.tests.testcase import TestCase # NOQA diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 0000000..0a610c2 --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,363 @@ +import logging +import unittest +import requests +import time +import os + +from logdna import LogDNAHandler +from concurrent.futures import ThreadPoolExecutor +from logdna.configs import defaults +from unittest import mock +from unittest.mock import patch + +now = int(time.time()) +expectedLines = [] +LOGDNA_API_KEY = os.environ.get('LOGDNA_INGESTION_KEY') +logger = logging.getLogger('logdna') +logger.setLevel(logging.INFO) +sample_args = { + 'app': 'differentTest', + 'level': 'debug', + 'hostname': 'differentHost', + 'env': 'differentEnv' +} + +sample_record = logging.LogRecord('test', logging.INFO, 'test', 5, + 'Something to test', [sample_args], '', '', + '') +sample_message = { + 'line': 'Something to test', + 'hostname': 'differentHost', + 'level': 'debug', + 'app': 'differentTest', + 'env': 'differentEnv', + 'meta': { + 'args': sample_args, + 'name': 'test', + 'pathname': 'test', + 'lineno': 5 + } +} + +sample_options = { + 'hostname': 'localhost', + 'ip': '10.0.1.1', + 'mac': 'C0:FF:EE:C0:FF:EE', + 'tags': 'sample,test', + 'index_meta': True, + 'now': int(time.time() * 1000), + 'retry_interval_secs': 0.5 +} + + +class MockThreadPoolExecutor(): + def __init__(self, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + pass + + def submit(self, fn, *args, **kwargs): + # execute functions in series without creating threads + # for easier unit testing + result = fn(*args, **kwargs) + return result + + def shutdown(self, wait=True): + pass + + +class LogDNAHandlerTest(unittest.TestCase): + def test_handler(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + self.assertIsInstance(handler, logging.Handler) + self.assertIsInstance( + handler.internal_handler, logging.StreamHandler) + self.assertIsNotNone(handler.internalLogger) + self.assertEqual(handler.key, LOGDNA_API_KEY) + self.assertEqual(handler.hostname, sample_options['hostname']) + self.assertEqual(handler.ip, sample_options['ip']) + self.assertEqual(handler.mac, sample_options['mac']) + self.assertEqual(handler.loglevel, 'info') + self.assertEqual(handler.app, '') + self.assertEqual(handler.env, '') + self.assertEqual(handler.tags, sample_options['tags'].split(',')) + self.assertEqual(handler.custom_fields, defaults['META_FIELDS']) + + # Set the Connection Variables + self.assertEqual(handler.url, defaults['LOGDNA_URL']) + self.assertEqual(handler.request_timeout, + defaults['DEFAULT_REQUEST_TIMEOUT']) + self.assertEqual(handler.user_agent, defaults['USER_AGENT']) + self.assertEqual(handler.max_retry_attempts, + defaults['MAX_RETRY_ATTEMPTS']) + self.assertEqual(handler.max_retry_jitter, + defaults['MAX_RETRY_JITTER']) + self.assertEqual(handler.max_concurrent_requests, + defaults['MAX_CONCURRENT_REQUESTS']) + self.assertEqual(handler.retry_interval_secs, + sample_options['retry_interval_secs']) + + # Set the Flush-related Variables + self.assertEqual(handler.buf, []) + self.assertEqual(handler.buf_size, 0) + self.assertIsNone(handler.flusher) + self.assertTrue(handler.index_meta) + self.assertEqual(handler.flush_limit, defaults['FLUSH_LIMIT']) + self.assertEqual(handler.flush_interval_secs, + defaults['FLUSH_INTERVAL_SECS']) + self.assertEqual(handler.buf_retention_limit, + defaults['BUF_RETENTION_LIMIT']) + + # Set up the Thread Pools + self.assertIsInstance( + handler.worker_thread_pool, ThreadPoolExecutor) + self.assertIsInstance( + handler.request_thread_pool, ThreadPoolExecutor) + self.assertEqual(handler.level, logging.DEBUG) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_flusher(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 200 + r.reason = 'OK' + post_mock.return_value = r + handler.emit(sample_record) + self.assertIsNotNone(handler.flusher) + handler.close_flusher() + self.assertIsNone(handler.flusher) + + def test_emit(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + handler.buffer_log = unittest.mock.Mock() + handler.emit(sample_record) + sample_message['timestamp'] = unittest.mock.ANY + handler.buffer_log.assert_called_once_with(sample_message) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_lock_and_do_flush_request(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 200 + r.reason = 'OK' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + test_buf = handler.buf.copy() + handler.try_lock_and_do_flush_request() + post_mock.assert_called_with( + url=handler.url, + json={ + 'e': 'ls', + 'ls': test_buf + }, + params={ + 'hostname': handler.hostname, + 'ip': handler.ip, + 'mac': handler.mac, + 'tags': handler.tags, + 'now': int(now * 1000) + }, + stream=True, + allow_redirects=True, + timeout=handler.request_timeout, + headers={ + 'user-agent': handler.user_agent, + 'apikey': LOGDNA_API_KEY}) + self.assertTrue(post_mock.call_count, 1) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_500(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 500 + r.reason = 'Internal Server Error' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 3) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_502(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 502 + r.reason = 'Bad Gateway' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 3) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_504(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 504 + r.reason = 'Gateway Timeout' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 3) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_429(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 429 + r.reason = 'Too Many Requests' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 3) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_403(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 403 + r.reason = 'Forbidden' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 1) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_403_log_response(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 403 + r.reason = 'Forbidden' + post_mock.return_value = r + sample_options['log_error_response'] = True + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 1) + + def test_close(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + close_flusher_mock = unittest.mock.Mock() + close_flusher_mock.side_effect = handler.close_flusher + handler.schedule_flush_sync = unittest.mock.Mock() + handler.close_flusher = close_flusher_mock + handler.close() + handler.close_flusher.assert_called_once_with() + handler.schedule_flush_sync.assert_called_once_with( + should_block=True) + self.assertIsNone(handler.worker_thread_pool) + self.assertIsNone(handler.request_thread_pool) + + # These should be separate objects, since there is already + # a variable in the base class named self.lock. We want + # to make sure that a separate lock is created for the + # locking semantics of the LogDNA Handler + def test_lock_var_separate_from_local_lock_var(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + self.assertIsNotNone(handler) + + # Test that we did not replace the base class' instance var. + self.assertIsNotNone(handler._lock) + self.assertIsNotNone(handler.lock) + self.assertNotEquals(handler.lock, handler._lock) + + def test_flush(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + handler.worker_thread_pool = MockThreadPoolExecutor() + handler.request_thread_pool = MockThreadPoolExecutor() + handler.buf = [sample_message] + handler.buf_size += len(handler.buf) + handler.try_request = unittest.mock.Mock() + handler.flush() + handler.try_request.assert_called_once_with([sample_message]) + + def test_buffer_log(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 200 + r.reason = 'OK' + post_mock.return_value = r + handler.worker_thread_pool = MockThreadPoolExecutor() + handler.request_thread_pool = MockThreadPoolExecutor() + handler.flush = unittest.mock.Mock() + sample_message['timestamp'] = now + handler.flush_limit = 0 + handler.buffer_log(sample_message) + handler.flush.assert_called_once_with() + self.assertEqual(handler.buf, [sample_message]) + self.assertEqual(handler.buf_size, len(sample_message['line'])) + + # Attempts to reproduce the specific scenario that resulted in + # https://mezmo.atlassian.net/browse/LOG-15414 where log messages + # would be dropped due to race conditions. The test essentially + # does the following: + # 1. Create a LogDNAHandler + # 2. Call handler.emit() with a large number of log records at a rate + # sufficiently high to trigger the race + # 3. Verify that no log records are dropped. + # + # This test is not deterministic, but it should be sufficient to + # catch regressions. It reliably reproduces the issue in question + # and fails with the previous version of this code. + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_when_emitManyLogs_then_noLogsDropped(self): + num_logs = 10**5 + received = list() + + def append_received(json=None, **kwargs): + ids = [int(log['line']) for log in json['ls']] + for id in ids: + received.append(id) + r = requests.Response() + r.status_code = 200 + r.reason = 'OK' + # Simulate some reasonable request latency + time.sleep(0.1) + return r + + def get_sample_record(id): + return logging.LogRecord( + name='test', + level=logging.INFO, + pathname='test', + lineno=5, + msg=str(id), + args=[sample_args], + exc_info='', + func='', + sinfo='') + + with patch('requests.post', side_effect=append_received): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + for i in range(num_logs): + handler.emit(get_sample_record(i)) + handler.close() + + self.assertEqual(len(received), num_logs) + self.assertEqual(set(received), set(range(num_logs))) + + def test_when_handlerShutDown_then_handlerDoesNotHang(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + self.assertIsNotNone(handler) + # Do nothing. This test should pass by virtue of not hanging. + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..446840f --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,61 @@ +import unittest +from unittest.mock import patch +from logdna.utils import is_jsonable +from logdna.utils import sanitize_meta +from logdna.utils import get_ip +from logdna.utils import normalize_list_option + +IP = '10.0.50.10' +VIP = '10.1.60.20' + + +class JSONTest(unittest.TestCase): + def setUp(self): + self.valid = {'key': 'value'} + self.invalid = {'key': set()} + + def test_serialize_valid_json(self): + self.assertTrue(is_jsonable(self.valid), 'json serializeble = True') + + def test_serialize_invalid_json(self): + self.assertFalse(is_jsonable(self.invalid), + 'non json serializable = False') + + +class SanitizeTest(unittest.TestCase): + def setUp(self): + self.valid = {'foo': 'bar', 'baz': 'whizbang'} + self.invalid = {'bar': 'foo', 'baz': set()} + + def test_sanitize_simple(self): + clean = sanitize_meta(self.valid, True) + self.assertDictEqual(clean, self.valid) + + def test_sanitize_complex(self): + clean = sanitize_meta(self.invalid, True) + self.assertDictEqual(clean, { + 'bar': 'foo', + '__errors': 'These keys have been sanitized: baz' + }) + + +class IPTest(unittest.TestCase): + @patch('socket.socket', **{'return_value.connect.side_effect': OSError()}) + def test_get_ip_socket_error(self, _): + self.assertEqual(get_ip(), '127.0.0.1', + 'default to localhost on error') + + @patch('socket.socket', + **{'return_value.getsockname.return_value': [IP, VIP]}) + def test_get_ip_default(self, _): + self.assertEqual(get_ip(), IP, 'default to localhost on error') + + +class NormalizeListOptionTest(unittest.TestCase): + def test_normalize_simple(self): + value1 = normalize_list_option({'tags': ' a, b'}, 'tags') + value2 = normalize_list_option({'tags': ['a', 'b']}, 'tags') + value3 = normalize_list_option({'tags': ('a', 'b')}, 'tags') + self.assertEqual(value1, ['a', 'b']) + self.assertEqual(value1, value2) + self.assertEqual(value3, [])