Skip to content

Commit

Permalink
fixes SIDN#172: Fix hadoop calls for non-existent users
Browse files Browse the repository at this point in the history
* if the user you're running as doesn't exist in passwd/nss the hadoop calls failed with "Caused by: javax.security.auth.login.LoginException: java.lang.NullPointerException: invalid null input: name"
  • Loading branch information
thomasdupas committed Mar 14, 2022
1 parent 8573647 commit 835ef8f
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 0 deletions.
11 changes: 11 additions & 0 deletions src/main/java/nl/sidnlabs/entrada/parquet/ParquetPartition.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Objects;
import java.util.UUID;
import org.apache.avro.Schema;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.parquet.avro.AvroParquetWriter;
import org.apache.parquet.column.ParquetProperties.WriterVersion;
import org.apache.parquet.hadoop.ParquetWriter;
Expand All @@ -17,6 +20,7 @@
import lombok.experimental.NonFinal;
import lombok.extern.log4j.Log4j2;
import nl.sidnlabs.entrada.exception.ApplicationException;
import org.springframework.beans.factory.annotation.Value;

@Log4j2
@Getter
Expand All @@ -25,6 +29,9 @@ public class ParquetPartition<T> {

private static Configuration conf = new Configuration();

@Value("${hadoop.login.user}")
private String loginUser;

private ParquetWriter<T> writer;
private String filename;
@NonFinal
Expand All @@ -33,6 +40,10 @@ public class ParquetPartition<T> {

public ParquetPartition(String partition, Schema schema, int rowgroupsize, int pageRowLimit) {

if (loginUser != null) {
UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(loginUser));
}

long start = System.currentTimeMillis();

currentFile = new Path(
Expand Down
4 changes: 4 additions & 0 deletions src/main/resources/application-dev.properties
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,10 @@ hdfs.username=hdfs
hdfs.data.owner=impala
hdfs.data.group=hive

# login user to use for hadoop calls, by default the UID gets looked up in /etc/passwd.
# This gives a nullpointer if it's not present, only overridden when value is not null
hadoop.login.user=

impala.log.level=3
impala.log.path=${entrada.location.log}/impala-logs/

Expand Down
4 changes: 4 additions & 0 deletions src/main/resources/application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,10 @@ hdfs.username=hdfs
hdfs.data.owner=impala
hdfs.data.group=hive

# login user to use for hadoop calls, by default the UID gets looked up in /etc/passwd.
# This gives a nullpointer if it's not present, only overridden when value is not null
hadoop.login.user=

impala.log.level=0
impala.log.path=${entrada.location.log}/impala-logs/

Expand Down

0 comments on commit 835ef8f

Please sign in to comment.