Improve BPE #1431

	name: OpenVINO tokenizers sdl tests
	on:
	workflow_dispatch:
	pull_request:
	merge_group:
	push:
	branches:
	- master
	- 'releases/**'

	permissions:
	actions: read
	contents: read
	security-events: write

	concurrency:
	# github.ref is not unique in post-commit
	group: ${{ github.event_name == 'push' && github.run_id \|\| github.ref }}-linux-sdl
	cancel-in-progress: true

	env:
	PYTHON_VERSION: '3.11'

	jobs:
	sdl_tests:
	name: SDL tests
	timeout-minutes: 10
	defaults:
	run:
	shell: bash
	runs-on: ubuntu-latest

	steps:
	- name: Clone Openvino tokenizers sources and tests
	uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1

	- name: Setup Python ${{ env.PYTHON_VERSION }}
	uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	cache: 'pip'

	- name: Install Python tests dependencies
	run: \|
	python3 -m pip install bandit

	- name: Tokenizers Bandit tests
	run: \|
	bandit -c pyproject.toml -r python

	- name: Run Trivy vulnerability scanner in fs mode
	uses: aquasecurity/trivy-action@a20de5420d57c4102486cdd9578b45609c99d7eb # v0.26.0
	with:
	scan-type: 'fs'
	scan-ref: '.'

	- name: Dependency Review
	if: ${{ github.event_name == 'pull_request' }}
	uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4
	with:
	config-file: './.github/dependency_review.yml'

Provide feedback