Skip to main content

Bitbucket (Self-Hosted)

In this example you are going to create a webhook integration between your Bitbucket Server and Port. The integration will facilitate the ingestion of Bitbucket project, repository and pull request entities into Port.

Port configuration​

Create the following blueprint definitions:

Bitbucket project blueprint
{
  "identifier": "bitbucketProject",
  "description": "A software catalog to represent Bitbucket project",
  "title": "Bitbucket Project",
  "icon": "BitBucket",
  "schema": {
    "properties": {
      "public": {
        "icon": "DefaultProperty",
        "title": "Public",
        "type": "boolean"
      },
      "description": {
        "title": "Description",
        "type": "string",
        "icon": "DefaultProperty"
      },
      "type": {
        "icon": "DefaultProperty",
        "title": "Type",
        "type": "string"
      },
      "link": {
        "title": "Link",
        "icon": "DefaultProperty",
        "type": "string"
      }
    },
    "required": []
  },
  "mirrorProperties": {},
  "calculationProperties": {},
  "relations": {}
}
Bitbucket repository blueprint
{
  "identifier": "service",
  "description": "A software catalog to represent Bitbucket repositories",
  "title": "Bitbucket Repository",
  "icon": "BitBucket",
  "schema": {
    "properties": {
      "forkable": {
        "icon": "DefaultProperty",
        "title": "Is Forkable",
        "type": "boolean"
      },
      "description": {
        "title": "Description",
        "type": "string",
        "icon": "DefaultProperty"
      },
      "public": {
        "icon": "DefaultProperty",
        "title": "Is Public",
        "type": "boolean"
      },
      "state": {
        "icon": "DefaultProperty",
        "title": "State",
        "type": "string"
      },
      "link": {
        "title": "Link",
        "icon": "DefaultProperty",
        "type": "string"
      }
    },
    "required": []
  },
  "mirrorProperties": {},
  "calculationProperties": {},
  "relations": {
    "project": {
      "title": "Project",
      "target": "bitbucketProject",
      "required": false,
      "many": false
    }
  }
}
Bitbucket pull request blueprint
{
  "identifier": "bitbucketPullrequest",
  "description": "A software catalog to represent Bitbucket pull requests",
  "title": "Bitbucket Pull Request",
  "icon": "BitBucket",
  "schema": {
    "properties": {
      "created_on": {
        "title": "Created On",
        "type": "string",
        "format": "date-time",
        "icon": "DefaultProperty"
      },
      "updated_on": {
        "title": "Updated On",
        "type": "string",
        "format": "date-time",
        "icon": "DefaultProperty"
      },
      "description": {
        "title": "Description",
        "type": "string",
        "icon": "DefaultProperty"
      },
      "state": {
        "icon": "DefaultProperty",
        "title": "State",
        "type": "string",
        "enum": ["OPEN", "MERGED", "DECLINED", "SUPERSEDED"],
        "enumColors": {
          "OPEN": "yellow",
          "MERGED": "green",
          "DECLINED": "red",
          "SUPERSEDED": "purple"
        }
      },
      "owner": {
        "title": "Owner",
        "type": "string",
        "icon": "DefaultProperty"
      },
      "link": {
        "title": "Link",
        "icon": "DefaultProperty",
        "type": "string"
      },
      "destination": {
        "title": "Destination Branch",
        "type": "string",
        "icon": "DefaultProperty"
      },
      "source": {
        "title": "Source Branch",
        "type": "string",
        "icon": "DefaultProperty"
      },
      "reviewers": {
        "items": {
          "type": "string"
        },
        "title": "Reviewers",
        "type": "array",
        "icon": "DefaultProperty"
      },
      "participants": {
        "items": {
          "type": "string"
        },
        "title": "Participants",
        "type": "array",
        "icon": "DefaultProperty"
      },
      "merge_commit": {
        "title": "Merge Commit",
        "type": "string",
        "icon": "DefaultProperty"
      }
    },
    "required": []
  },
  "mirrorProperties": {},
  "calculationProperties": {},
  "relations": {
    "repository": {
      "title": "Repository",
      "target": "service",
      "required": false,
      "many": false
    }
  }
}
Blueprint Properties

You may modify the properties in your blueprints depending on what you want to track in your Bitbucket account.

Create the following webhook configuration using Port's UI

Bitbucket webhook configuration

  1. Basic details tab - fill the following details:

    1. Title : Bitbucket Server Mapper;
    2. Identifier : bitbucket_server_mapper;
    3. Description : A webhook configuration to map Bitbucket projects, repositories and pull requests to Port;
    4. Icon : BitBucket;

  2. Integration configuration tab - fill the following JQ mapping:

    [
      {
        "blueprint": "bitbucketProject",
        "filter": ".body.eventKey == \"project:modified\"",
        "entity": {
          "identifier": ".body.new.key | tostring",
          "title": ".body.new.name",
          "properties": {
            "public": ".body.new.public",
            "type": ".body.new.type",
            "description": ".body.new.description",
            "link": ".body.new.links.self[0].href"
          }
        }
      },
      {
        "blueprint": "service",
        "filter": ".body.eventKey == \"repo:modified\"",
        "entity": {
          "identifier": ".body.new.slug",
          "title": ".body.new.name",
          "properties": {
            "description": ".body.new.description",
            "state": ".body.new.state",
            "forkable": ".body.new.forkable",
            "public": ".body.new.public",
            "link": ".body.new.links.self[0].href"
          },
          "relations": {
            "project": ".body.new.project.key"
          }
        }
      },
      {
        "blueprint": "bitbucketPullrequest",
        "filter": ".body.eventKey | startswith(\"pr:\")",
        "entity": {
          "identifier": ".body.pullRequest.id | tostring",
          "title": ".body.pullRequest.title",
          "properties": {
            "created_on": ".body.pullRequest.createdDate | (tonumber / 1000 | strftime(\"%Y-%m-%dT%H:%M:%SZ\"))",
            "updated_on": ".body.pullRequest.updatedDate | (tonumber / 1000 | strftime(\"%Y-%m-%dT%H:%M:%SZ\"))",
            "merge_commit": ".body.pullRequest.fromRef.latestCommit",
            "state": ".body.pullRequest.state",
            "owner": ".body.pullRequest.author.user.displayName",
            "link": ".body.pullRequest.links.self[0].href",
            "destination": ".body.pullRequest.toRef.displayId",
            "source": ".body.pullRequest.fromRef.displayId",
            "participants": "[.body.pullRequest.participants[].user.displayName]",
            "reviewers": "[.body.pullRequest.reviewers[].user.displayName]"
          },
          "relations": {
            "repository": ".body.pullRequest.toRef.repository.slug"
          }
        }
      }
    ]
    note

    Take note of, and copy the Webhook URL that is provided in this tab

  3. Click Save at the bottom of the page.

Create a webhook in Bitbucket​

  1. From your Bitbucket account, open the project where you want to add the webhook;
  2. Click Project settings or the gear icon on the left sidebar;
  3. On the Workflow section, select Webhooks on the left sidebar;
  4. Click the Add webhook button to create a webhook for the repository;
  5. Input the following details:
    1. Title - use a meaningful name such as Port Webhook;
    2. URL - enter the value of the webhook URL you received after creating the webhook configuration in Port;
    3. Secret - enter the value of the secret you provided when configuring the webhook in Port;
    4. Triggers -
      1. Under Project select modified;
      2. Under Repository select modified;
      3. Under Pull request select any event based on your use case.
  6. Click Save to save the webhook;
tip

Follow this documentation to learn more about webhook events payload in Bitbucket.

Done! any change that happens to your project, repository or pull requests in Bitbucket will trigger a webhook event to the webhook URL provided by Port. Port will parse the events according to the mapping and update the catalog entities accordingly.

Let's Test It​

This section includes a sample webhook event sent from Bitbucket when a pull request is merged. In addition, it includes the entity created from the event based on the webhook configuration provided in the previous section.

Payload​

Here is an example of the payload structure sent to the webhook URL when a Bitbucket pull request is merged:

Webhook event payload
{
  "body": {
    "eventKey": "pr:merged",
    "date": "2023-11-16T11:03:42+0000",
    "actor": {
      "name": "admin",
      "emailAddress": "username@gmail.com",
      "active": true,
      "displayName": "Test User",
      "id": 2,
      "slug": "admin",
      "type": "NORMAL",
      "links": {
        "self": [
          {
            "href": "http://myhost:7990/users/admin"
          }
        ]
      }
    },
    "pullRequest": {
      "id": 2,
      "version": 2,
      "title": "lint code",
      "description": "here is the description",
      "state": "MERGED",
      "open": false,
      "closed": true,
      "createdDate": 1700132280533,
      "updatedDate": 1700132622026,
      "closedDate": 1700132622026,
      "fromRef": {
        "id": "refs/heads/dev",
        "displayId": "dev",
        "latestCommit": "9e08604e14fa72265d65696608725c2b8f7850f2",
        "type": "BRANCH",
        "repository": {
          "slug": "data-analyses",
          "id": 1,
          "name": "data analyses",
          "description": "This is for my repository and all the blah blah blah",
          "hierarchyId": "24cfae4b0dd7bade7edc",
          "scmId": "git",
          "state": "AVAILABLE",
          "statusMessage": "Available",
          "forkable": true,
          "project": {
            "key": "MOPP",
            "id": 1,
            "name": "My On Prem Project",
            "description": "On premise test project is sent to us for us",
            "public": false,
            "type": "NORMAL",
            "links": {
              "self": [
                {
                  "href": "http://myhost:7990/projects/MOPP"
                }
              ]
            }
          },
          "public": false,
          "archived": false,
          "links": {
            "clone": [
              {
                "href": "ssh://git@myhost:7999/mopp/data-analyses.git",
                "name": "ssh"
              },
              {
                "href": "http://myhost:7990/scm/mopp/data-analyses.git",
                "name": "http"
              }
            ],
            "self": [
              {
                "href": "http://myhost:7990/projects/MOPP/repos/data-analyses/browse"
              }
            ]
          }
        }
      },
      "toRef": {
        "id": "refs/heads/main",
        "displayId": "main",
        "latestCommit": "e461aae894b6dc951f405dca027a3f5567ea6bee",
        "type": "BRANCH",
        "repository": {
          "slug": "data-analyses",
          "id": 1,
          "name": "data analyses",
          "description": "This is for my repository and all the blah blah blah",
          "hierarchyId": "24cfae4b0dd7bade7edc",
          "scmId": "git",
          "state": "AVAILABLE",
          "statusMessage": "Available",
          "forkable": true,
          "project": {
            "key": "MOPP",
            "id": 1,
            "name": "My On Prem Project",
            "description": "On premise test project is sent to us for us",
            "public": false,
            "type": "NORMAL",
            "links": {
              "self": [
                {
                  "href": "http://myhost:7990/projects/MOPP"
                }
              ]
            }
          },
          "public": false,
          "archived": false,
          "links": {
            "clone": [
              {
                "href": "ssh://git@myhost:7999/mopp/data-analyses.git",
                "name": "ssh"
              },
              {
                "href": "http://myhost:7990/scm/mopp/data-analyses.git",
                "name": "http"
              }
            ],
            "self": [
              {
                "href": "http://myhost:7990/projects/MOPP/repos/data-analyses/browse"
              }
            ]
          }
        }
      },
      "locked": false,
      "author": {
        "user": {
          "name": "admin",
          "emailAddress": "username@gmail.com",
          "active": true,
          "displayName": "Test User",
          "id": 2,
          "slug": "admin",
          "type": "NORMAL",
          "links": {
            "self": [
              {
                "href": "http://myhost:7990/users/admin"
              }
            ]
          }
        },
        "role": "AUTHOR",
        "approved": false,
        "status": "UNAPPROVED"
      },
      "reviewers": [],
      "participants": [],
      "properties": {
        "mergeCommit": {
          "displayId": "1cbccf99220",
          "id": "1cbccf99220b23f89624c7c604f630663a1aaf8e"
        }
      },
      "links": {
        "self": [
          {
            "href": "http://myhost:7990/projects/MOPP/repos/data-analyses/pull-requests/2"
          }
        ]
      }
    }
  },
  "headers": {
    "X-Forwarded-For": "10.0.148.57",
    "X-Forwarded-Proto": "https",
    "X-Forwarded-Port": "443",
    "Host": "ingest.getport.io",
    "X-Amzn-Trace-Id": "Self=1-6555f719-267a0fce1e7a4d8815de94f7;Root=1-6555f719-1906872f41621b17250bb83a",
    "Content-Length": "2784",
    "User-Agent": "Atlassian HttpClient 3.0.4 / Bitbucket-8.15.1 (8015001) / Default",
    "Content-Type": "application/json; charset=UTF-8",
    "accept": "*/*",
    "X-Event-Key": "pr:merged",
    "X-Hub-Signature": "sha256=bf366faf8d8c41a4af21d25d922b87c3d1d127b5685238b099d2f311ad46e978",
    "X-Request-Id": "d5fa6a16-bb6c-40d6-9c50-bc4363e79632",
    "via": "HTTP/1.1 AmazonAPIGateway",
    "forwarded": "for=154.160.30.235;host=ingest.getport.io;proto=https"
  },
  "queryParams": {}
}

Mapping Result​

{
   "identifier":"2",
   "title":"lint code",
   "blueprint":"bitbucketPullrequest",
   "properties":{
      "created_on":"2023-11-16T10:58:00Z",
      "updated_on":"2023-11-16T11:03:42Z",
      "merge_commit":"9e08604e14fa72265d65696608725c2b8f7850f2",
      "state":"MERGED",
      "owner":"Test User",
      "link":"http://myhost:7990/projects/MOPP/repos/data-analyses/pull-requests/2",
      "destination":"main",
      "source":"dev",
      "participants":[],
      "reviewers":[]
   },
   "relations":{
      "repository":"data-analyses"
   },
   "filter":true
}

Import Bitbucket Historical Issues​

In this example you are going to use the provided Python script to fetch data from the Bitbucket Server API and ingest it to Port.

Prerequisites​

This example utilizes the same blueprint and webhook definition from the previous section.

In addition, provide the following environment variables:

  • PORT_CLIENT_ID - Your Port client id
  • PORT_CLIENT_SECRET - Your Port client secret
  • BITBUCKET_HOST - Bitbucket server host such as http://localhost:7990
  • BITBUCKET_USERNAME - Bitbucket username to use when accessing the Bitbucket resources
  • BITBUCKET_PASSWORD - Bitbucket account password
info

Find your Port credentials using this guide

Use the following Python script to ingest historical Bitbucket projects, repositories and pull requests into port:

Bitbucket Python script

## Import the needed libraries
import requests
from requests.auth import HTTPBasicAuth
from decouple import config
from loguru import logger
from typing import Any
import time
from datetime import datetime


# Get environment variables using the config object or os.environ["KEY"]

PORT_CLIENT_ID = config("PORT_CLIENT_ID")
PORT_CLIENT_SECRET = config("PORT_CLIENT_SECRET")
BITBUCKET_USERNAME = config("BITBUCKET_USERNAME")
BITBUCKET_PASSWORD = config("BITBUCKET_PASSWORD")
BITBUCKET_API_URL = config("BITBUCKET_HOST")
PORT_API_URL = "https://api.getport.io/v1"

## According to https://support.atlassian.com/bitbucket-cloud/docs/api-request-limits/
RATE_LIMIT = 1000  # Maximum number of requests allowed per hour
RATE_PERIOD = 3600  # Rate limit reset period in seconds (1 hour)

# Initialize rate limiting variables
request_count = 0
rate_limit_start = time.time()

## Get Port Access Token
credentials = {'clientId': PORT_CLIENT_ID, 'clientSecret': PORT_CLIENT_SECRET}
token_response = requests.post(f'{PORT_API_URL}/auth/access_token', json=credentials)
access_token = token_response.json()['accessToken']

# You can now use the value in access_token when making further requests
port_headers = {
	'Authorization': f'Bearer {access_token}'
}

## Bitbucket user password https://developer.atlassian.com/server/bitbucket/how-tos/example-basic-authentication/
bitbucket_auth = HTTPBasicAuth(username=BITBUCKET_USERNAME, password=BITBUCKET_PASSWORD)


def add_entity_to_port(blueprint_id, entity_object):
    response = requests.post(f'{PORT_API_URL}/blueprints/{blueprint_id}/entities?upsert=true&merge=true', json=entity_object, headers=port_headers)
    logger.info(response.json())

def get_paginated_resource(path: str, params: dict[str, Any] = None, page_size: int = 25):
    logger.info(f"Requesting data for {path}")

    global request_count, rate_limit_start

    # Check if we've exceeded the rate limit, and if so, wait until the reset period is over
    if request_count >= RATE_LIMIT:
        elapsed_time = time.time() - rate_limit_start
        if elapsed_time < RATE_PERIOD:
            sleep_time = RATE_PERIOD - elapsed_time
            time.sleep(sleep_time)

        # Reset the rate limiting variables
        request_count = 0
        rate_limit_start = time.time()

    url = f"{BITBUCKET_API_URL}/rest/api/1.0/{path}"
    params = params or {}
    params["limit"] = page_size
    next_page_start = None

    while True:
        try:
            if next_page_start:
                params["start"] = next_page_start

            response = requests.get(url=url, auth=bitbucket_auth, params=params)
            response.raise_for_status()
            page_json = response.json()
            request_count += 1
            batch_data = page_json["values"]
            yield batch_data

            # Check for next page start in response
            next_page_start = page_json.get("nextPageStart")

            # Break the loop if there is no more data
            if not next_page_start:
                break
        except requests.exceptions.HTTPError as e:
            logger.error(f"HTTP error with code {e.response.status_code}, content: {e.response.text}")
            raise
    logger.info(f"Successfully fetched paginated data for {path}")

def convert_to_datetime(timestamp: int):
    converted_datetime = datetime.utcfromtimestamp(timestamp / 1000.0)
    return converted_datetime.strftime('%Y-%m-%dT%H:%M:%SZ')

def process_project_entities(projects_data: list[dict[str, Any]]):
    blueprint_id = "bitbucketProject"

    for project in projects_data:
        entity = {
            "identifier": project["key"],
            "title": project["name"],
            "properties": {
                "description": project.get("description"),
                "public": project["public"],
                "type": project["type"],
                "link": project["links"]["self"][0]["href"]
            },
            "relations": {}
        }
        add_entity_to_port(blueprint_id=blueprint_id, entity_object=entity)

def process_repository_entities(repository_data: list[dict[str, Any]]):
    blueprint_id = "service"

    for repo in repository_data:

        entity = {
        "identifier": repo["slug"],
        "title": repo["name"],
        "properties": {
            "description": repo.get("description"),
            "state": repo["state"],
            "forkable": repo["forkable"],
            "public": repo["public"],
            "link": repo["links"]["self"][0]["href"]
        },
        "relations": {
            "project": repo["project"]["key"]
        }
        }
        add_entity_to_port(blueprint_id=blueprint_id, entity_object=entity)

def process_pullrequest_entities(pullrequest_data: list[dict[str, Any]]):
    blueprint_id = "bitbucketPullrequest"

    for pr in pullrequest_data:

        entity = {
            "identifier": str(pr["id"]),
            "title": pr["title"],
            "properties": {
                "created_on": convert_to_datetime(pr["createdDate"]),
                "updated_on": convert_to_datetime(pr["updatedDate"]),
                "merge_commit": pr["fromRef"]["latestCommit"],
                "description": pr.get("description"),
                "state": pr["state"],
                "owner": pr["author"]["user"]["displayName"],
                "link": pr["links"]["self"][0]["href"],
                "destination": pr["toRef"]["displayId"],
                "participants": [user["user"]["displayName"] for user in pr.get("participants", [])],
                "reviewers": [user["user"]["displayName"] for user in pr.get("reviewers", [])],
                "source": pr["fromRef"]["displayId"]
            },
            "relations": {
                "repository": pr["toRef"]["repository"]["slug"]
            }
        }
        add_entity_to_port(blueprint_id=blueprint_id, entity_object=entity)


def get_repositories(project: dict[str, Any]):
    repositories_path = f"projects/{project['key']}/repos"
    for repositories_batch in get_paginated_resource(path=repositories_path):
        logger.info(f"received repositories batch with size {len(repositories_batch)} from project: {project['key']}")
        process_repository_entities(repository_data=repositories_batch)

        get_repository_pull_requests(repository_batch=repositories_batch)


def get_repository_pull_requests(repository_batch: list[dict[str, Any]]):
    pr_params = {"state": "ALL"} ## Fetch all pull requests
    for repository in repository_batch:
        pull_requests_path = f"projects/{repository['project']['key']}/repos/{repository['slug']}/pull-requests"
        for pull_requests_batch in get_paginated_resource(path=pull_requests_path, params=pr_params):
            logger.info(f"received pull requests batch with size {len(pull_requests_batch)} from repo: {repository['slug']}")
            process_pullrequest_entities(pullrequest_data=pull_requests_batch)

if __name__ == "__main__":
    project_path = "projects"
    for projects_batch in get_paginated_resource(path=project_path):
        logger.info(f"received projects batch with size {len(projects_batch)}")
        process_project_entities(projects_data=projects_batch)

        for project in projects_batch:
            get_repositories(project=project)

Done! you are now able to import historical projects, repositories and pull requests from Bitbucket server into Port. Port will parse the objects according to the mapping and update the catalog entities accordingly.