From c7f74b592f98c496b0043c12b8594bc665cf5801 Mon Sep 17 00:00:00 2001 From: jdrean Date: Tue, 17 Oct 2023 15:50:57 +0200 Subject: [PATCH] SR - fix the duplicate of mutiple event types in the same topic --- docs/content-template.md | 3 +- .../README.md | 19 +++++ .../start.sh | 1 - .../README.md | 51 ------------- .../start.sh | 74 ------------------- .../stop.sh | 8 -- .../multiple-event-types-in-topic/README.md | 0 .../alltypes.avsc | 0 .../customer.avsc | 0 .../product.avsc | 0 .../start-avro.sh | 0 .../multiple-event-types-in-topic/stop.sh | 0 12 files changed, 20 insertions(+), 136 deletions(-) delete mode 100644 schema-registry/multiple-event-type-same-topic-api/README.md delete mode 100644 schema-registry/multiple-event-type-same-topic-api/start.sh delete mode 100644 schema-registry/multiple-event-type-same-topic-api/stop.sh rename {other => schema-registry}/multiple-event-types-in-topic/README.md (100%) rename {other => schema-registry}/multiple-event-types-in-topic/alltypes.avsc (100%) rename {other => schema-registry}/multiple-event-types-in-topic/customer.avsc (100%) rename {other => schema-registry}/multiple-event-types-in-topic/product.avsc (100%) rename {other => schema-registry}/multiple-event-types-in-topic/start-avro.sh (100%) mode change 100755 => 100644 rename {other => schema-registry}/multiple-event-types-in-topic/stop.sh (100%) mode change 100755 => 100644 diff --git a/docs/content-template.md b/docs/content-template.md index 97bdeb310..06b8a1c68 100644 --- a/docs/content-template.md +++ b/docs/content-template.md @@ -323,7 +323,7 @@ There is also a Confluent Cloud version available [here](https://github.com/vdes ## 📝 Schema Registry - [How to produce Avro records via kafka-avro-console-producer with Union](https://github.com/vdesabou/kafka-docker-playground/tree/master/schema-registry/kafka-avro-console-producer-union) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet) -- [Multiple event types in the same topic via Schema Registry API](https://github.com/vdesabou/kafka-docker-playground/tree/master/schema-registry/multiple-event-type-same-topic-api) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet) +- [Multiple Event Types in the Same Topic](https://github.com/vdesabou/kafka-docker-playground/tree/master/schema-registry/multiple-event-types-in-topic) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet) - [How to use different Key and Value serializer with kafka-avro-console-producer](https://github.com/vdesabou/kafka-docker-playground/tree/master/schema-registry/use-diffrent-key-value-serializer-console-producer) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet) ## 👾 Other Playgrounds @@ -337,7 +337,6 @@ There is also a Confluent Cloud version available [here](https://github.com/vdes - 👻 Testing [KIP-108](https://cwiki.apache.org/confluence/display/KAFKA/KIP-108%3A+Create+Topic+Policy) [Create Topic Policy](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/create-topic-policy) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet) - 📉 [Monitoring Demo](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/monitoring-demo) ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet) - 🕵️‍♀️ [Kafka Connect Sink Monitoring Demo](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/monitoring-sink-latency) Showcase different Kafka Connect Sink troubleshooting scenarios ![not tested](https://img.shields.io/badge/CI-not%20tested!-violet) -- 🎫 [Multiple Event Types in the Same Topic](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/multiple-event-types-in-topic) :other/multiple-event-types-in-topic: - 👊 [Integrate syslogs to detect SSH failure connections](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/syslog-logstash-ksqldb) using Syslog source connector, LogStash and ksqlDB :other/syslog-logstash-ksqldb: - 📶 [How to ensure high availability of LDAP using DNS SRV Records](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/ldap-authorizer-with-ldap-failover) :other/ldap-authorizer-with-ldap-failover: - 🕹 [AVRO examples](https://github.com/vdesabou/kafka-docker-playground/tree/master/other/schema-format-avro) including a JAVA producer :other/schema-format-avro: diff --git a/schema-registry/kafka-avro-console-producer-union/README.md b/schema-registry/kafka-avro-console-producer-union/README.md index 60ff47a74..2afbc2886 100644 --- a/schema-registry/kafka-avro-console-producer-union/README.md +++ b/schema-registry/kafka-avro-console-producer-union/README.md @@ -5,6 +5,25 @@ With Apache Avro, you can set a field as an UNION of two types. A union datatype is used whenever the field has one or more datatypes. For example, ["null", "string"] declares a schema which may be either a null or string. Union type can be useful for Optional fields: +``` +{ + "type":"record", + "name":"myrecord", + "fields":[ + { "name":"id", "type":"int"}, + { "name":"product", "type":"string"}, + { "name":"quantity", "type":"int"}, + { + "name":"description", + "type":[ + "null", + "string" + ], + "default":null + } + ] +} +``` But it can be also useful for putting several Event Types in the same topic: ``` diff --git a/schema-registry/kafka-avro-console-producer-union/start.sh b/schema-registry/kafka-avro-console-producer-union/start.sh index 567710e70..972a87bf6 100644 --- a/schema-registry/kafka-avro-console-producer-union/start.sh +++ b/schema-registry/kafka-avro-console-producer-union/start.sh @@ -27,7 +27,6 @@ docker exec -i connect kafka-avro-console-consumer --bootstrap-server broker:909 # CASE 2 -> With Mutiple Event Types in the same Topic - log "Register the Avro schema for Customer" # { # "type": "record", diff --git a/schema-registry/multiple-event-type-same-topic-api/README.md b/schema-registry/multiple-event-type-same-topic-api/README.md deleted file mode 100644 index aa49ffe9a..000000000 --- a/schema-registry/multiple-event-type-same-topic-api/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Putting Several Event Types in the Same Topic with Schema Registry API - -## Objective - -We register a first Avro schema for Product: -``` -{ - "type": "record", - "namespace": "io.confluent.examples.avro", - "name": "Product", - "fields": [ - {"name": "product_id", "type": "int"}, - {"name": "product_name", "type": "string"}, - {"name": "product_price", "type": "double"} - ] -} -``` -and the schema for Customer: -``` -{ - "type": "record", - "namespace": "io.confluent.examples.avro", - "name": "Customer", - "fields": [ - { "name": "customer_id", "type": "int" }, - { "name": "customer_name", "type": "string" }, - { "name": "customer_email", "type": "string" }, - { "name": "customer_address", "type": "string" } - ] -} -``` -Then we can register a new schema for topic alltypes with Schema References to these two schemas: -``` -curl -s -H "Content-Type: application/vnd.schemaregistry.v1+json" \ - -X POST http://schema-registry:8081/subjects/alltypes-value/versions \ - --data '{"schema":"[\"io.confluent.examples.avro.Customer\",\"io.confluent.examples.avro.Product\"]","schemaType":"AVRO","references":[{"name":"io.confluent.examples.avro.Customer","subject":"customer-value","version":1},{"name":"io.confluent.examples.avro.Product","subject":"product-value","version":1}]}' -``` - - -To learn more about Schema Reference, see the example given below in [Multiple event types in the same topic](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/serdes-avro.html#multiple-event-types-same-topic-avro), the associated [blog post](https://www.confluent.io/blog/multiple-event-types-in-the-same-kafka-topic/), and the API example for how to register (create) a new schema in [POST /subjects/(string: subject)/versions](https://docs.confluent.io/platform/7.5/schema-registry/develop/api.html#api-example-avro-schema-references) - -## How to run - -Simply run: - -``` -$ playground run -f start -``` - -## Resources -https://www.confluent.io/blog/multiple-event-types-in-the-same-kafka-topic diff --git a/schema-registry/multiple-event-type-same-topic-api/start.sh b/schema-registry/multiple-event-type-same-topic-api/start.sh deleted file mode 100644 index 37c1a4e80..000000000 --- a/schema-registry/multiple-event-type-same-topic-api/start.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -set -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -source ${DIR}/../../scripts/utils.sh - -${DIR}/../../environment/plaintext/start.sh - -log "Register the Avro schema for Customer" -# { -# "type": "record", -# "namespace": "io.confluent.examples.avro", -# "name": "Customer", -# "fields": [ -# { "name": "customer_id", "type": "int" }, -# { "name": "customer_name", "type": "string" }, -# { "name": "customer_email", "type": "string" }, -# { "name": "customer_address", "type": "string" } -# ] -# } -docker exec -i connect curl -s -H "Content-Type: application/vnd.schemaregistry.v1+json" \ - -X POST http://schema-registry:8081/subjects/customer-value/versions \ - --data '{"schema":"{\"type\":\"record\",\"namespace\":\"io.confluent.examples.avro\",\"name\":\"Customer\",\"fields\":[{\"name\":\"customer_id\",\"type\":\"int\"},{\"name\":\"customer_name\",\"type\":\"string\"},{\"name\":\"customer_email\",\"type\":\"string\"},{\"name\":\"customer_address\",\"type\":\"string\"}]}"}' - -log "Register the Avro schema for Product" -# { -# "type": "record", -# "namespace": "io.confluent.examples.avro", -# "name": "Product", -# "fields": [ -# {"name": "product_id", "type": "int"}, -# {"name": "product_name", "type": "string"}, -# {"name": "product_price", "type": "double"} -# ] -# } -docker exec -i connect curl -s -H "Content-Type: application/vnd.schemaregistry.v1+json" \ - -X POST http://schema-registry:8081/subjects/product-value/versions \ - --data '{"schema":"{\"type\":\"record\",\"namespace\":\"io.confluent.examples.avro\",\"name\":\"Product\",\"fields\":[{\"name\":\"product_id\",\"type\":\"int\"},{\"name\":\"product_name\",\"type\":\"string\"},{\"name\":\"product_price\",\"type\":\"double\"}]}"}' - -log "Register the Avro schema for AllTypes" -# { -# "schema":"[\"io.confluent.examples.avro.Customer\",\"io.confluent.examples.avro.Product\"]", -# "schemaType":"AVRO", -# "references":[ -# { -# "name":"io.confluent.examples.avro.Customer", -# "subject":"customer-value", -# "version":1 -# }, -# { -# "name":"io.confluent.examples.avro.Product", -# "subject":"product-value", -# "version":1 -# } -# ] -# } -docker exec -i connect curl -s -H "Content-Type: application/vnd.schemaregistry.v1+json" \ - -X POST http://schema-registry:8081/subjects/alltypes-value/versions \ - --data '{"schema":"[\"io.confluent.examples.avro.Customer\",\"io.confluent.examples.avro.Product\"]","schemaType":"AVRO","references":[{"name":"io.confluent.examples.avro.Customer","subject":"customer-value","version":1},{"name":"io.confluent.examples.avro.Product","subject":"product-value","version":1}]}' - -log "Produce records to alltypes topic" -# Configure the Avro serializer to use your Avro union for serialization, and not the event type, by configuring the following properties in your producer application: -# auto.register.schemas=false -# use.latest.version=true -docker exec -i connect kafka-avro-console-producer --broker-list broker:9092 --property schema.registry.url=http://schema-registry:8081 --topic alltypes --property value.schema.id=3 --property auto.register=false --property use.latest.version=true << EOF -{ "io.confluent.examples.avro.Product": { "product_id": 1, "product_name" : "rice", "product_price" : 100.00 } } -{ "io.confluent.examples.avro.Customer": { "customer_id": 100, "customer_name": "acme", "customer_email": "acme@google.com", "customer_address": "1 Main St" } } -EOF - -log "Consuming records from this topic" -docker exec -i connect kafka-avro-console-consumer --bootstrap-server broker:9092 \ - --topic alltypes --from-beginning \ - --property schema.registry.url=http://schema-registry:8081 --property print.schema.ids=true --property schema.id.separator=: \ - --max-messages 2 diff --git a/schema-registry/multiple-event-type-same-topic-api/stop.sh b/schema-registry/multiple-event-type-same-topic-api/stop.sh deleted file mode 100644 index bab6807da..000000000 --- a/schema-registry/multiple-event-type-same-topic-api/stop.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - - - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -source ${DIR}/../../scripts/utils.sh - -stop_all "$DIR" \ No newline at end of file diff --git a/other/multiple-event-types-in-topic/README.md b/schema-registry/multiple-event-types-in-topic/README.md similarity index 100% rename from other/multiple-event-types-in-topic/README.md rename to schema-registry/multiple-event-types-in-topic/README.md diff --git a/other/multiple-event-types-in-topic/alltypes.avsc b/schema-registry/multiple-event-types-in-topic/alltypes.avsc similarity index 100% rename from other/multiple-event-types-in-topic/alltypes.avsc rename to schema-registry/multiple-event-types-in-topic/alltypes.avsc diff --git a/other/multiple-event-types-in-topic/customer.avsc b/schema-registry/multiple-event-types-in-topic/customer.avsc similarity index 100% rename from other/multiple-event-types-in-topic/customer.avsc rename to schema-registry/multiple-event-types-in-topic/customer.avsc diff --git a/other/multiple-event-types-in-topic/product.avsc b/schema-registry/multiple-event-types-in-topic/product.avsc similarity index 100% rename from other/multiple-event-types-in-topic/product.avsc rename to schema-registry/multiple-event-types-in-topic/product.avsc diff --git a/other/multiple-event-types-in-topic/start-avro.sh b/schema-registry/multiple-event-types-in-topic/start-avro.sh old mode 100755 new mode 100644 similarity index 100% rename from other/multiple-event-types-in-topic/start-avro.sh rename to schema-registry/multiple-event-types-in-topic/start-avro.sh diff --git a/other/multiple-event-types-in-topic/stop.sh b/schema-registry/multiple-event-types-in-topic/stop.sh old mode 100755 new mode 100644 similarity index 100% rename from other/multiple-event-types-in-topic/stop.sh rename to schema-registry/multiple-event-types-in-topic/stop.sh