Skip to content

Commit

Permalink
Added JSON serialize SMT (#1)
Browse files Browse the repository at this point in the history
  • Loading branch information
zloyuser authored Aug 25, 2023
1 parent 472caf9 commit a48f50b
Show file tree
Hide file tree
Showing 16 changed files with 1,626 additions and 1 deletion.
5 changes: 5 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#
# https://help.github.com/articles/dealing-with-line-endings/
#
# These are explicitly windows files and should use crlf
*.bat text eol=crlf
36 changes: 36 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: Release

on:
push:
tags:
- 'v*'

jobs:
build-project:
runs-on: ubuntu-latest
steps:
- name: Checkout project sources
uses: actions/checkout@v2
- name: Setup Gradle
uses: gradle/gradle-build-action@v2
- name: Build project
run: ./gradlew build
- name: Create release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
with:
tag_name: ${{ github.ref }}
release_name: ${{ github.ref }}
draft: true
prerelease: false
- name: Upload artifact
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./build/libs/onliner-kafka-smt-1.0.0.jar
asset_name: onliner-kafka-smt.jar
asset_content_type: application/java-archive
17 changes: 17 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
name: Tests

on:
push:
branches:
- '*'

jobs:
run-tests:
runs-on: ubuntu-latest
steps:
- name: Checkout project sources
uses: actions/checkout@v2
- name: Setup Gradle
uses: gradle/gradle-build-action@v2
- name: Run tests
run: ./gradlew test
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Ignore Gradle project-specific cache directory
.gradle
# Ignore Gradle build output directory
build
lib/build
.idea
21 changes: 21 additions & 0 deletions LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
The MIT License (MIT)

Copyright (c) Onliner

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
39 changes: 38 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1 +1,38 @@
# kafka-smt
# Onliner Transformations for Apache Kafka® Connect

A collection of [Single Message Transformations (SMTs)](https://kafka.apache.org/documentation/#connect_transforms) for Apache Kafka Connect.

## Transformations

See [the Kafka documentation](https://kafka.apache.org/documentation/#connect_transforms) for more details about configuring transformations.

### `JsonSerialize`

This transformation serialize part of the original record's data to JSON strings.

The transformation:
- expects the record value/key to be either a `STRUCT` or a `MAP`;
- expects it to have a specified field;

Exists in two variants:
- `org.onliner.kafka.transforms.JsonSerialize$Key` - works on keys;
- `org.onliner.kafka.transforms.JsonSerialize$Value` - works on values.

The transformation defines the following configurations:
- `fields` - List of fields to serialize. Cannot be `null` or empty.

- Here's an example of this transformation configuration:

```properties
transforms=encode
transforms.encode.type=org.onliner.kafka.transforms.JsonSerialize$Value
transforms.encode.fields=comma,separated,list,of,fields
```

## License

This project is licensed under the [MIT license](LICENSE).

## Trademarks

Apache Kafka and Apache Kafka Connect are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries.
102 changes: 102 additions & 0 deletions build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/*
* User Manual available at https://docs.gradle.org/7.4/userguide/building_java_projects.html
*/

group = "org.onliner.kafka.smt"
version = System.getenv("VERSION") ?: "1.0.0"

val javaVersion = 11

val artifactoryContext =
project.properties.getOrDefault("artifactory_context", System.getenv("ARTIFACTORY_CONTEXT")).toString()
val artifactoryUsern =
project.properties.getOrDefault("artifactory_user", System.getenv("ARTIFACTORY_USER")).toString()
val artifactoryPassword =
project.properties.getOrDefault("artifactory_password", System.getenv("ARTIFACTORY_PWD")).toString()

plugins {
kotlin("jvm") version "1.6.21"
idea // Generates files that are used by IntelliJ IDEA, thus making it possible to open the project from IDEA
`java-library` // Apply the java-library plugin for API and implementation separation.
`maven-publish`
id("io.gitlab.arturbosch.detekt") version "1.20.0"
}

repositories {
mavenCentral()
}

dependencies {
val kafkaConnectVersion = "3.2.+"
val junitVersion = "5.8.2"

compileOnly(platform("org.jetbrains.kotlin:kotlin-bom")) // Align versions of all Kotlin components
compileOnly("org.jetbrains.kotlin:kotlin-stdlib-jdk8") // Use the Kotlin JDK 8 standard library.

implementation("org.apache.kafka:connect-api:$kafkaConnectVersion")
implementation("org.apache.kafka:connect-json:$kafkaConnectVersion")
implementation("org.apache.kafka:connect-transforms:$kafkaConnectVersion")

testImplementation("org.jetbrains.kotlin:kotlin-test-junit")
testImplementation("org.junit.jupiter:junit-jupiter-api:$junitVersion")
}

kotlin {
jvmToolchain {
(this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of(javaVersion))
}
}

publishing {
publications {
create<MavenPublication>("maven") {
groupId = project.group.toString()
artifactId = project.name
version = project.version.toString()

pom {
name.set("onliner-kafka-smt")
description.set( project.description)
url.set("https://github.com/onliner/kafka-smt")
organization {
name.set("Onliner")
url.set("https://github.com/onliner")
}
issueManagement {
system.set("GitHub")
url.set("https://github.com/onliner/kafka-smt/issues")
}
licenses {
license {
name.set( "Apache License 2.0")
url.set("https://github.com/onliner/kafka-smt/blob/master/LICENSE")
distribution.set("repo")
}
}
developers {
developer {
name.set("onliner")
}
}
scm {
url.set("https://github.com/onliner/kafka-smt")
connection.set("scm:git:git://github.com/onliner/kafka-smt.git")
developerConnection.set("scm:git:ssh://[email protected]:onliner/kafka-smt.git")
}
}

from(components["java"])
}
}
repositories {
maven {
name = "ArtifactoryLocal"
url = uri(artifactoryContext + "/libs-release-local")
credentials {
username = artifactoryUsern
password = artifactoryPassword
}
}
}
}

Loading

0 comments on commit a48f50b

Please sign in to comment.