diff options
| author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
| commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
| tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /vdslib/src/test | |
Publish
Diffstat (limited to 'vdslib/src/test')
16 files changed, 2193 insertions, 0 deletions
diff --git a/vdslib/src/test/files/documentlist-java.dat b/vdslib/src/test/files/documentlist-java.dat Binary files differnew file mode 100644 index 00000000000..f4a097f9595 --- /dev/null +++ b/vdslib/src/test/files/documentlist-java.dat diff --git a/vdslib/src/test/files/documentmanager.cfg b/vdslib/src/test/files/documentmanager.cfg new file mode 100644 index 00000000000..cab4d3bfef1 --- /dev/null +++ b/vdslib/src/test/files/documentmanager.cfg @@ -0,0 +1,91 @@ +enablecompression false +datatype[3] +datatype[0].id -1007405344 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name benchmark.header +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].field[15] +datatype[0].structtype[0].field[0].name seed +datatype[0].structtype[0].field[0].id[0] +datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[1].name totaldoccount +datatype[0].structtype[0].field[1].id[0] +datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[2].name totalusercount +datatype[0].structtype[0].field[2].id[0] +datatype[0].structtype[0].field[2].datatype 0 +datatype[0].structtype[0].field[3].name minheaderdocsize +datatype[0].structtype[0].field[3].id[0] +datatype[0].structtype[0].field[3].datatype 0 +datatype[0].structtype[0].field[4].name maxheaderdocsize +datatype[0].structtype[0].field[4].id[0] +datatype[0].structtype[0].field[4].datatype 0 +datatype[0].structtype[0].field[5].name minbodydocsize +datatype[0].structtype[0].field[5].id[0] +datatype[0].structtype[0].field[5].datatype 0 +datatype[0].structtype[0].field[6].name maxbodydocsize +datatype[0].structtype[0].field[6].id[0] +datatype[0].structtype[0].field[6].datatype 0 +datatype[0].structtype[0].field[7].name simplecontent +datatype[0].structtype[0].field[7].id[0] +datatype[0].structtype[0].field[7].datatype 16 +datatype[0].structtype[0].field[8].name pregeneratedcontent +datatype[0].structtype[0].field[8].id[0] +datatype[0].structtype[0].field[8].datatype 16 +datatype[0].structtype[0].field[9].name headerint +datatype[0].structtype[0].field[9].id[0] +datatype[0].structtype[0].field[9].datatype 0 +datatype[0].structtype[0].field[10].name headerfloat +datatype[0].structtype[0].field[10].id[0] +datatype[0].structtype[0].field[10].datatype 1 +datatype[0].structtype[0].field[11].name headerstring +datatype[0].structtype[0].field[11].id[0] +datatype[0].structtype[0].field[11].datatype 2 +datatype[0].structtype[0].field[12].name headerraw +datatype[0].structtype[0].field[12].id[0] +datatype[0].structtype[0].field[12].datatype 3 +datatype[0].structtype[0].field[13].name headerlong +datatype[0].structtype[0].field[13].id[0] +datatype[0].structtype[0].field[13].datatype 4 +datatype[0].structtype[0].field[14].name headerdouble +datatype[0].structtype[0].field[14].id[0] +datatype[0].structtype[0].field[14].datatype 5 +datatype[0].documenttype[0] +datatype[1].id -851747595 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name benchmark.body +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[6] +datatype[1].structtype[0].field[0].name bodyint +datatype[1].structtype[0].field[0].id[0] +datatype[1].structtype[0].field[0].datatype 0 +datatype[1].structtype[0].field[1].name bodyfloat +datatype[1].structtype[0].field[1].id[0] +datatype[1].structtype[0].field[1].datatype 1 +datatype[1].structtype[0].field[2].name bodystring +datatype[1].structtype[0].field[2].id[0] +datatype[1].structtype[0].field[2].datatype 2 +datatype[1].structtype[0].field[3].name bodyraw +datatype[1].structtype[0].field[3].id[0] +datatype[1].structtype[0].field[3].datatype 3 +datatype[1].structtype[0].field[4].name bodylong +datatype[1].structtype[0].field[4].id[0] +datatype[1].structtype[0].field[4].datatype 4 +datatype[1].structtype[0].field[5].name bodydouble +datatype[1].structtype[0].field[5].id[0] +datatype[1].structtype[0].field[5].datatype 5 +datatype[1].documenttype[0] +datatype[2].id 2132196223 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[1] +datatype[2].documenttype[0].name benchmark +datatype[2].documenttype[0].version 0 +datatype[2].documenttype[0].inherits[0] +datatype[2].documenttype[0].headerstruct -1007405344 +datatype[2].documenttype[0].bodystruct -851747595 diff --git a/vdslib/src/test/files/documenttypes.cfg b/vdslib/src/test/files/documenttypes.cfg new file mode 100644 index 00000000000..d1c208f3a08 --- /dev/null +++ b/vdslib/src/test/files/documenttypes.cfg @@ -0,0 +1,126 @@ +enablecompression false +documenttype[1] +documenttype[0].id 2132196223 +documenttype[0].name "benchmark" +documenttype[0].version 0 +documenttype[0].headerstruct -1007405344 +documenttype[0].bodystruct -851747595 +documenttype[0].inherits[0] +documenttype[0].datatype[2] +documenttype[0].datatype[0].id -1007405344 +documenttype[0].datatype[0].type STRUCT +documenttype[0].datatype[0].array.element.id 0 +documenttype[0].datatype[0].map.key.id 0 +documenttype[0].datatype[0].map.value.id 0 +documenttype[0].datatype[0].wset.key.id 0 +documenttype[0].datatype[0].wset.createifnonexistent false +documenttype[0].datatype[0].wset.removeifzero false +documenttype[0].datatype[0].annotationref.annotation.id 0 +documenttype[0].datatype[0].sstruct.name "benchmark.header" +documenttype[0].datatype[0].sstruct.version 0 +documenttype[0].datatype[0].sstruct.compression.type NONE +documenttype[0].datatype[0].sstruct.compression.level 0 +documenttype[0].datatype[0].sstruct.compression.threshold 90 +documenttype[0].datatype[0].sstruct.compression.minsize 0 +documenttype[0].datatype[0].sstruct.field[15] +documenttype[0].datatype[0].sstruct.field[0].name "headerdouble" +documenttype[0].datatype[0].sstruct.field[0].id 225925695 +documenttype[0].datatype[0].sstruct.field[0].id_v6 1880905482 +documenttype[0].datatype[0].sstruct.field[0].datatype 5 +documenttype[0].datatype[0].sstruct.field[1].name "headerfloat" +documenttype[0].datatype[0].sstruct.field[1].id 1625470423 +documenttype[0].datatype[0].sstruct.field[1].id_v6 1473207887 +documenttype[0].datatype[0].sstruct.field[1].datatype 1 +documenttype[0].datatype[0].sstruct.field[2].name "headerint" +documenttype[0].datatype[0].sstruct.field[2].id 762302378 +documenttype[0].datatype[0].sstruct.field[2].id_v6 20914570 +documenttype[0].datatype[0].sstruct.field[2].datatype 0 +documenttype[0].datatype[0].sstruct.field[3].name "headerlong" +documenttype[0].datatype[0].sstruct.field[3].id 1173959018 +documenttype[0].datatype[0].sstruct.field[3].id_v6 447443892 +documenttype[0].datatype[0].sstruct.field[3].datatype 4 +documenttype[0].datatype[0].sstruct.field[4].name "headerraw" +documenttype[0].datatype[0].sstruct.field[4].id 391586263 +documenttype[0].datatype[0].sstruct.field[4].id_v6 1004217617 +documenttype[0].datatype[0].sstruct.field[4].datatype 3 +documenttype[0].datatype[0].sstruct.field[5].name "headerstring" +documenttype[0].datatype[0].sstruct.field[5].id 788916026 +documenttype[0].datatype[0].sstruct.field[5].id_v6 104714882 +documenttype[0].datatype[0].sstruct.field[5].datatype 2 +documenttype[0].datatype[0].sstruct.field[6].name "maxbodydocsize" +documenttype[0].datatype[0].sstruct.field[6].id 1712672954 +documenttype[0].datatype[0].sstruct.field[6].id_v6 843532182 +documenttype[0].datatype[0].sstruct.field[6].datatype 0 +documenttype[0].datatype[0].sstruct.field[7].name "maxheaderdocsize" +documenttype[0].datatype[0].sstruct.field[7].id 1706483157 +documenttype[0].datatype[0].sstruct.field[7].id_v6 2053499986 +documenttype[0].datatype[0].sstruct.field[7].datatype 0 +documenttype[0].datatype[0].sstruct.field[8].name "minbodydocsize" +documenttype[0].datatype[0].sstruct.field[8].id 655737763 +documenttype[0].datatype[0].sstruct.field[8].id_v6 1790708929 +documenttype[0].datatype[0].sstruct.field[8].datatype 0 +documenttype[0].datatype[0].sstruct.field[9].name "minheaderdocsize" +documenttype[0].datatype[0].sstruct.field[9].id 138730086 +documenttype[0].datatype[0].sstruct.field[9].id_v6 1021372057 +documenttype[0].datatype[0].sstruct.field[9].datatype 0 +documenttype[0].datatype[0].sstruct.field[10].name "pregeneratedcontent" +documenttype[0].datatype[0].sstruct.field[10].id 1358420191 +documenttype[0].datatype[0].sstruct.field[10].id_v6 107210372 +documenttype[0].datatype[0].sstruct.field[10].datatype 16 +documenttype[0].datatype[0].sstruct.field[11].name "seed" +documenttype[0].datatype[0].sstruct.field[11].id 1584656448 +documenttype[0].datatype[0].sstruct.field[11].id_v6 2028175984 +documenttype[0].datatype[0].sstruct.field[11].datatype 0 +documenttype[0].datatype[0].sstruct.field[12].name "simplecontent" +documenttype[0].datatype[0].sstruct.field[12].id 1986822798 +documenttype[0].datatype[0].sstruct.field[12].id_v6 1673731589 +documenttype[0].datatype[0].sstruct.field[12].datatype 16 +documenttype[0].datatype[0].sstruct.field[13].name "totaldoccount" +documenttype[0].datatype[0].sstruct.field[13].id 428258253 +documenttype[0].datatype[0].sstruct.field[13].id_v6 400080120 +documenttype[0].datatype[0].sstruct.field[13].datatype 0 +documenttype[0].datatype[0].sstruct.field[14].name "totalusercount" +documenttype[0].datatype[0].sstruct.field[14].id 1328467779 +documenttype[0].datatype[0].sstruct.field[14].id_v6 1421225290 +documenttype[0].datatype[0].sstruct.field[14].datatype 0 +documenttype[0].datatype[1].id -851747595 +documenttype[0].datatype[1].type STRUCT +documenttype[0].datatype[1].array.element.id 0 +documenttype[0].datatype[1].map.key.id 0 +documenttype[0].datatype[1].map.value.id 0 +documenttype[0].datatype[1].wset.key.id 0 +documenttype[0].datatype[1].wset.createifnonexistent false +documenttype[0].datatype[1].wset.removeifzero false +documenttype[0].datatype[1].annotationref.annotation.id 0 +documenttype[0].datatype[1].sstruct.name "benchmark.body" +documenttype[0].datatype[1].sstruct.version 0 +documenttype[0].datatype[1].sstruct.compression.type NONE +documenttype[0].datatype[1].sstruct.compression.level 0 +documenttype[0].datatype[1].sstruct.compression.threshold 90 +documenttype[0].datatype[1].sstruct.compression.minsize 0 +documenttype[0].datatype[1].sstruct.field[6] +documenttype[0].datatype[1].sstruct.field[0].name "bodydouble" +documenttype[0].datatype[1].sstruct.field[0].id 409250413 +documenttype[0].datatype[1].sstruct.field[0].id_v6 896473220 +documenttype[0].datatype[1].sstruct.field[0].datatype 5 +documenttype[0].datatype[1].sstruct.field[1].name "bodyfloat" +documenttype[0].datatype[1].sstruct.field[1].id 627746455 +documenttype[0].datatype[1].sstruct.field[1].id_v6 644475212 +documenttype[0].datatype[1].sstruct.field[1].datatype 1 +documenttype[0].datatype[1].sstruct.field[2].name "bodyint" +documenttype[0].datatype[1].sstruct.field[2].id 1128436051 +documenttype[0].datatype[1].sstruct.field[2].id_v6 1680592860 +documenttype[0].datatype[1].sstruct.field[2].datatype 0 +documenttype[0].datatype[1].sstruct.field[3].name "bodylong" +documenttype[0].datatype[1].sstruct.field[3].id 1947207690 +documenttype[0].datatype[1].sstruct.field[3].id_v6 82043035 +documenttype[0].datatype[1].sstruct.field[3].datatype 4 +documenttype[0].datatype[1].sstruct.field[4].name "bodyraw" +documenttype[0].datatype[1].sstruct.field[4].id 864880572 +documenttype[0].datatype[1].sstruct.field[4].id_v6 2105006012 +documenttype[0].datatype[1].sstruct.field[4].datatype 3 +documenttype[0].datatype[1].sstruct.field[5].name "bodystring" +documenttype[0].datatype[1].sstruct.field[5].id 316734404 +documenttype[0].datatype[1].sstruct.field[5].id_v6 474444010 +documenttype[0].datatype[1].sstruct.field[5].datatype 2 +documenttype[0].annotationtype[0] diff --git a/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java new file mode 100644 index 00000000000..33b8dd39e55 --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib;
+
+import com.yahoo.document.BucketId;
+
+import java.util.Random;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class BucketDistributionTestCase extends junit.framework.TestCase {
+
+ private static final int NUM_COLUMNS = 16;
+ private static final int MIN_BUCKETBITS = 4;
+ private static final int MAX_BUCKETBITS = 9;
+
+ public void printExpected() {
+ System.out.println(" int[][] expected = new int[][] {");
+ for (int numBucketBits = MIN_BUCKETBITS; numBucketBits <= MAX_BUCKETBITS; ++numBucketBits) {
+ System.out.print(" new int[] {");
+ BucketDistribution bd = new BucketDistribution(NUM_COLUMNS, numBucketBits);
+ for (int i = 0; i < bd.getNumBuckets(); ++i) {
+ if (i % 32 == 0) {
+ System.out.println("");
+ System.out.print(" ");
+ }
+ System.out.print(bd.getColumn(new BucketId(16, i)));
+ if (i < bd.getNumBuckets() - 1) {
+ System.out.print(", ");
+ }
+ }
+ System.out.print(" }");
+ if (numBucketBits < MAX_BUCKETBITS) {
+ System.out.print(",");
+ }
+ System.out.println("");
+ }
+ System.out.println(" };");
+ }
+
+ public void testDistribution() {
+ int[][] expected = new int[][] {
+ new int[] {
+ 10, 11, 9, 6, 4, 8, 14, 1, 13, 2, 12, 3, 5, 7, 15, 0 },
+ new int[] {
+ 11, 12, 11, 13, 8, 13, 8, 9, 4, 5, 6, 12, 10, 15, 1, 1, 7, 9, 14, 2, 2, 14, 3, 3, 4, 5, 6, 7, 10, 15, 0, 0 },
+ new int[] {
+ 13, 13, 13, 13, 9, 11, 8, 9, 11, 14, 9, 11, 14, 14, 8, 10, 11, 14, 4, 5, 5, 6, 6, 7, 8, 10, 12, 15, 1, 1, 1, 1,
+ 6, 7, 8, 10, 12, 15, 2, 2, 2, 2, 12, 15, 3, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7, 7, 9, 10, 12, 15, 0, 0, 0, 0 },
+ new int[] {
+ 14, 14, 14, 13, 11, 14, 14, 10, 13, 14, 10, 12, 8, 8, 9, 10, 9, 10, 11, 12, 13, 15, 11, 12, 13, 13, 15, 8, 8, 9, 10, 11,
+ 11, 12, 13, 15, 4, 4, 5, 5, 5, 5, 15, 6, 6, 7, 7, 7, 8, 9, 9, 10, 11, 12, 14, 15, 1, 1, 1, 1, 1, 1, 1, 1,
+ 6, 6, 6, 7, 7, 8, 8, 9, 10, 11, 12, 13, 15, 2, 2, 2, 2, 2, 2, 2, 2, 12, 13, 15, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 9, 9, 10, 11, 12, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0 },
+ new int[] {
+ 15, 14, 15, 15, 12, 14, 15, 12, 12, 11, 12, 13, 14, 13, 13, 13, 14, 15, 15, 9, 14, 9, 15, 10, 11, 11, 12, 8, 8, 8, 8, 9,
+ 9, 10, 10, 10, 11, 11, 12, 13, 13, 14, 10, 11, 11, 12, 13, 13, 14, 13, 13, 14, 15, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, 11,
+ 10, 10, 11, 11, 12, 13, 13, 14, 15, 4, 4, 4, 4, 15, 5, 5, 5, 5, 5, 5, 5, 15, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7,
+ 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 14, 15, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 14, 15, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 12, 13, 14, 14, 15, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7,
+ 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+ new int[] {
+ 15, 15, 15, 15, 14, 14, 15, 13, 13, 13, 13, 13, 14, 14, 15, 12, 14, 15, 15, 11, 11, 12, 13, 13, 13, 14, 14, 15, 15, 10, 12, 12,
+ 12, 13, 13, 13, 14, 14, 15, 15, 9, 9, 9, 10, 10, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 15, 15, 8, 8, 8, 8, 9, 9, 9,
+ 9, 9, 9, 10, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 14, 14, 10, 10, 10, 11, 11, 11, 12, 12, 12, 12,
+ 13, 13, 14, 14, 14, 15, 15, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,
+ 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 4, 4, 4, 4, 4, 4, 4, 15, 15, 5, 5, 5, 5, 5,
+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 14, 14, 14, 15, 15, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 15, 15, 15,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9,
+ 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15,
+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5,
+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+ 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 15, 15, 15,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+ };
+
+ for (int numBucketBits = MIN_BUCKETBITS; numBucketBits <= MAX_BUCKETBITS; ++numBucketBits) {
+ BucketDistribution bd = new BucketDistribution(NUM_COLUMNS, numBucketBits);
+ assertEquals(NUM_COLUMNS, bd.getNumColumns());
+ assertEquals(1 << numBucketBits, bd.getNumBuckets());
+ for (int i = 0; i < bd.getNumBuckets(); ++i) {
+ assertEquals(expected[numBucketBits - MIN_BUCKETBITS][i], bd.getColumn(new BucketId(16, i)));
+ }
+ }
+ }
+
+ public void testNumBucketBits() {
+ Random rnd = new Random();
+
+ BucketDistribution bd = new BucketDistribution(1, 4);
+ for (int i = 0; i <= 0xf; ++i) {
+ assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 4) & i)));
+ }
+
+ bd = new BucketDistribution(1, 8);
+ for (int i = 0; i <= 0xff; ++i) {
+ assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 8) & i)));
+ }
+
+ bd = new BucketDistribution(1, 16);
+ for (int i = 0; i <= 0xffff; ++i) {
+ assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 16) & i)));
+ }
+ }
+}
diff --git a/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java new file mode 100644 index 00000000000..fd7ffbc409d --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.document.update.FieldUpdate; + +import java.io.EOFException; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class DocumentListTestCase extends junit.framework.TestCase { + + @SuppressWarnings("deprecation") + public void testSelfSerializationAndWriteJavaFile() throws Exception { + DocumentTypeManager docMan = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(docMan, "file:src/test/files/documentmanager.cfg"); + + DocumentType bmType = docMan.getDocumentType("benchmark"); + DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); + put1.getDocument().setFieldValue("headerstring", "foo"); + DocumentRemove doc2 = new DocumentRemove(new DocumentId("userdoc:foo:99999999:2")); + DocumentPut put3 = new DocumentPut(bmType, "userdoc:foo:99999999:3"); + put3.getDocument().setFieldValue("bodyfloat", new FloatFieldValue(5.5f)); + + + DocumentUpdate docUp = new DocumentUpdate(docMan.getDocumentType("benchmark"), new DocumentId("userdoc:foo:99999999:4")); + docUp.addFieldUpdate(FieldUpdate.createAssign(docUp.getType().getField("bodystring"), new StringFieldValue("ballooooo"))); + + List<Entry> entries = new ArrayList<Entry>(); + entries.add(Entry.create(put1)); + entries.add(Entry.create(doc2)); + entries.add(Entry.create(put3)); + entries.add(Entry.create(docUp)); + + DocumentList documentList = DocumentList.create(entries); + + DocumentSerializer gbuf = DocumentSerializerFactory.create42(); + gbuf.putInt(null, 1234); // Add some data to avoid special case where position() is 0 for buffer used. + int startPos = gbuf.getBuf().position(); + documentList.serialize(gbuf); + + int size = gbuf.getBuf().position() - startPos; + byte[] data = new byte[size]; + gbuf.getBuf().position(startPos); + gbuf.getBuf().get(data); + FileOutputStream stream = new FileOutputStream("./src/test/files/documentlist-java.dat"); + stream.write(data); + stream.close(); + + gbuf.getBuf().position(0); + + DocumentList documentList2 = DocumentList.create(docMan, data); + + assertEquals(4, documentList2.size()); + Entry entry1 = documentList2.get(0); + assertEquals(0L, entry1.getTimestamp()); + assertFalse(entry1.isBodyStripped()); + assertFalse(entry1.isRemoveEntry()); + assertFalse(entry1.isUpdateEntry()); + assertTrue(entry1.getDocumentOperation() instanceof DocumentPut); + assertEquals(new StringFieldValue("foo"), ((DocumentPut) entry1.getDocumentOperation()).getDocument().getFieldValue("headerstring")); + + Entry entry2 = documentList2.get(1); + assertEquals(0L, entry2.getTimestamp()); + assertFalse(entry2.isBodyStripped()); + assertTrue(entry2.isRemoveEntry()); + assertFalse(entry2.isUpdateEntry()); + assertTrue(entry2.getDocumentOperation() instanceof DocumentRemove); + + Entry entry3 = documentList2.get(2); + assertEquals(0L, entry3.getTimestamp()); + assertFalse(entry3.isBodyStripped()); + assertFalse(entry3.isRemoveEntry()); + assertFalse(entry3.isUpdateEntry()); + assertTrue(entry3.getDocumentOperation() instanceof DocumentPut); + assertEquals(new FloatFieldValue(5.5f), ((DocumentPut) entry3.getDocumentOperation()).getDocument().getFieldValue("bodyfloat")); + + Entry entry4 = documentList2.get(3); + assertEquals(0L, entry4.getTimestamp()); + assertFalse(entry4.isBodyStripped()); + assertFalse(entry4.isRemoveEntry()); + assertTrue(entry4.isUpdateEntry()); + assertTrue(entry4.getDocumentOperation() instanceof DocumentUpdate); + assertEquals(new StringFieldValue("ballooooo"),((DocumentUpdate) entry4.getDocumentOperation()).getFieldUpdate(0).getValueUpdate(0).getValue()); + } + + public void testContains() { + DocumentTypeManager manager = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(manager, "file:src/test/files/documentmanager.cfg"); + + DocumentType bmType = manager.getDocumentType("benchmark"); + DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); + DocumentRemove remove1 = new DocumentRemove(new DocumentId("userdoc:foo:99999999:2")); + DocumentPut put2 = new DocumentPut(bmType, "userdoc:foo:99999999:3"); + + List<Entry> entries = new ArrayList<Entry>(); + entries.add(Entry.create(put1)); + entries.add(Entry.create(remove1)); + entries.add(Entry.create(put2)); + + DocumentList documentList = DocumentList.create(entries); + + DocumentPut put3 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); + DocumentRemove remove2 = new DocumentRemove(new DocumentId("userdoc:foo:99999999:2")); + DocumentPut put4 = new DocumentPut(bmType, "userdoc:foo:99999999:3"); + + List<Entry> entries2 = new ArrayList<Entry>(); + entries2.add(Entry.create(put3)); + entries2.add(Entry.create(remove2)); + entries2.add(Entry.create(put4)); + + DocumentList documentList2 = DocumentList.create(entries2); + + assert(documentList.containsAll(documentList2)); + + Long t = put4.getDocument().getLastModified(); + put4.getDocument().setLastModified(13L); + assert(!documentList.containsAll(documentList2)); + put4.getDocument().setLastModified(t); + + assert(documentList.containsAll(documentList2)); + + entries.remove(2); + assert(!documentList.containsAll(documentList2)); + + } +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java new file mode 100644 index 00000000000..2f1140e32b6 --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; + +/** + * @author banino + */ +public class EntryTestCase extends junit.framework.TestCase{ + + public void testEquals() { + DocumentTypeManager manager = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(manager, "file:src/test/files/documentmanager.cfg"); + + DocumentType bmType = manager.getDocumentType("benchmark"); + DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); + DocumentPut put2 = new DocumentPut(bmType, "userdoc:foo:99999999:2"); + + Entry entry1 = Entry.create(put1); + Entry entry2 = Entry.create(put1); + assert(entry1.equals(entry2)); + + Entry entry3 = Entry.create(put2); + assert(!entry1.equals(entry3)); + + } + + public void testHashCode() { + DocumentTypeManager manager = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(manager, "file:src/test/files/documentmanager.cfg"); + + DocumentType bmType = manager.getDocumentType("benchmark"); + DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); + DocumentPut put2 = new DocumentPut(bmType, "userdoc:foo:99999999:2"); + + Entry entry1 = Entry.create(put1); + Entry entry2 = Entry.create(put1); + assert(entry1.hashCode() == entry2.hashCode()); + + Entry entry3 = Entry.create(put2); + assert(entry1.hashCode() != entry3.hashCode()); + + } + + + +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/SearchResultTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/SearchResultTestCase.java new file mode 100644 index 00000000000..eea51bf7787 --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/SearchResultTestCase.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author balder + */ +public class SearchResultTestCase { + + @Test + public void requireThatHitsOrderWell() { + SearchResult.Hit a = new SearchResult.Hit("a", 0); + SearchResult.Hit b = new SearchResult.Hit("b", 0.1); + SearchResult.Hit c = new SearchResult.Hit("c", 1.0); + SearchResult.Hit bb = new SearchResult.Hit("b2", 0.1); + assertTrue(a.compareTo(a) == 0); + assertTrue(a.compareTo(b) > 0); + assertTrue(a.compareTo(c) > 0); + assertTrue(b.compareTo(a) < 0); + assertTrue(b.compareTo(bb) == 0); + assertTrue(bb.compareTo(b) == 0); + assertTrue(b.compareTo(c) > 0); + assertTrue(c.compareTo(a) < 0); + assertTrue(c.compareTo(b) < 0); + + byte [] b1 = {0x00}; + byte [] b2 = {0x07}; + byte [] b3 = {0x7f}; + byte [] b4 = {(byte)0x80}; + byte [] b5 = {(byte)0xb1}; + byte [] b6 = {(byte)0xff}; + + assertEquals(0x00, b1[0]); + assertEquals(0x07, b2[0]); + assertEquals(0x7f, b3[0]); + assertEquals(0x80, ((int)b4[0]) & 0xff); + assertEquals(0xb1, ((int)b5[0]) & 0xff); + assertEquals(0xff, ((int)b6[0]) & 0xff); + SearchResult.Hit h1 = new SearchResult.HitWithSortBlob(a, b1); + SearchResult.Hit h2 = new SearchResult.HitWithSortBlob(a, b2); + SearchResult.Hit h3 = new SearchResult.HitWithSortBlob(a, b3); + SearchResult.Hit h4 = new SearchResult.HitWithSortBlob(a, b4); + SearchResult.Hit h5 = new SearchResult.HitWithSortBlob(a, b5); + SearchResult.Hit h6 = new SearchResult.HitWithSortBlob(a, b6); + + assertTrue(h1.compareTo(h1) == 0); + assertTrue(h1.compareTo(h2) < 0); + assertTrue(h1.compareTo(h3) < 0); + assertTrue(h1.compareTo(h4) < 0); + assertTrue(h1.compareTo(h5) < 0); + assertTrue(h1.compareTo(h6) < 0); + + assertTrue(h2.compareTo(h1) > 0); + assertTrue(h2.compareTo(h2) == 0); + assertTrue(h2.compareTo(h3) < 0); + assertTrue(h2.compareTo(h4) < 0); + assertTrue(h2.compareTo(h5) < 0); + assertTrue(h2.compareTo(h6) < 0); + + assertTrue(h3.compareTo(h1) > 0); + assertTrue(h3.compareTo(h2) > 0); + assertTrue(h3.compareTo(h3) == 0); + assertTrue(h3.compareTo(h4) < 0); + assertTrue(h3.compareTo(h5) < 0); + assertTrue(h3.compareTo(h6) < 0); + + assertTrue(h4.compareTo(h1) > 0); + assertTrue(h4.compareTo(h2) > 0); + assertTrue(h4.compareTo(h3) > 0); + assertTrue(h4.compareTo(h4) == 0); + assertTrue(h4.compareTo(h5) < 0); + assertTrue(h4.compareTo(h6) < 0); + + assertTrue(h5.compareTo(h1) > 0); + assertTrue(h5.compareTo(h2) > 0); + assertTrue(h5.compareTo(h3) > 0); + assertTrue(h5.compareTo(h4) > 0); + assertTrue(h5.compareTo(h5) == 0); + assertTrue(h5.compareTo(h6) < 0); + + assertTrue(h6.compareTo(h1) > 0); + assertTrue(h6.compareTo(h2) > 0); + assertTrue(h6.compareTo(h3) > 0); + assertTrue(h6.compareTo(h4) > 0); + assertTrue(h6.compareTo(h5) > 0); + assertTrue(h6.compareTo(h6) == 0); + } +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/VisitorOrderingTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/VisitorOrderingTestCase.java new file mode 100644 index 00000000000..ed8f7d44d0d --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/VisitorOrderingTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib; + +import org.junit.Test; + +import static org.junit.Assert.*; + +public class VisitorOrderingTestCase { + + @Test + public void testVisitorOrderingDefault() { + VisitorOrdering ordering = new VisitorOrdering(); + assertEquals(VisitorOrdering.ASCENDING, ordering.getOrder()); + assertEquals(0, ordering.getDivisionBits()); + assertEquals(0, ordering.getWidthBits()); + assertEquals(0, ordering.getOrderingStart()); + assertEquals("+,0,0,0", ordering.toString()); + } + + @Test + public void testVisitorOrderingAscending() { + VisitorOrdering ordering = new VisitorOrdering(VisitorOrdering.ASCENDING); + assertEquals(VisitorOrdering.ASCENDING, ordering.getOrder()); + assertEquals(0, ordering.getDivisionBits()); + assertEquals(0, ordering.getWidthBits()); + assertEquals(0, ordering.getOrderingStart()); + assertEquals("+,0,0,0", ordering.toString()); + } + + + @Test + public void testVisitorOrderingComplex() { + VisitorOrdering ordering = new VisitorOrdering(VisitorOrdering.DESCENDING, (long)3, (short)2, (short)1); + assertEquals(VisitorOrdering.DESCENDING, ordering.getOrder()); + assertEquals(1, ordering.getDivisionBits()); + assertEquals(2, ordering.getWidthBits()); + assertEquals(3, ordering.getOrderingStart()); + assertEquals("-,2,1,3", ordering.toString()); + } +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java new file mode 100644 index 00000000000..2c28bfd1f8b --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib.distribution; + +import java.io.*; + +/** + * Helper class to implement unit tests that should produce the same result in different implementations. + */ +public abstract class CrossPlatformTestFactory { + private final String directory; + private final String name; + + public CrossPlatformTestFactory(String directory, String name) { + this.directory = directory; + this.name = name; + } + + public String getName() { return name; } + + public boolean loadTestResults() throws Exception { + File reference = new File(directory, name + ".reference.results"); + if (!reference.exists()) { + return false; + } + BufferedReader br = new BufferedReader(new FileReader(reference)); + StringBuilder sb = new StringBuilder(); + try{ + while(true) { + String line = br.readLine(); + if (line == null) break; + sb.append(line); + } + parse(sb.toString()); + } finally { + br.close(); + } + return true; + } + + public void recordTestResults() throws Exception { + File results = new File(directory, name + ".java.results"); + FileWriter fw = new FileWriter(results); + try{ + fw.write(serialize()); + } finally { + fw.close(); + } + } + + public abstract String serialize() throws Exception; + public abstract void parse(String serialized) throws Exception; +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java new file mode 100644 index 00000000000..cdc7f8a5dd3 --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java @@ -0,0 +1,387 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib.distribution; + +import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.document.BucketId; +import com.yahoo.vdslib.state.NodeType; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.Stack; + +public class DistributionTestCase extends junit.framework.TestCase { + private DistributionTestFactory test; + /** Build a set of buckets to test that should represent the entire bucket space well. */ + private static List<BucketId> getTestBuckets() { return getTestBuckets(16); } + private static List<BucketId> getTestBuckets(int minUsedBits) { + List<BucketId> buckets = new ArrayList<>(); + assertTrue(minUsedBits <= 16); + // Get a set of buckets from the same split level + for (int i=16; i<=18; ++i) { + for (int j=0; j<20; ++j) { + buckets.add(new BucketId(i, j)); + } + } + // Get a few random buckets at every split level. + Random randomized = new Random(413); + long randValue = randomized.nextLong(); + for (int i=minUsedBits; i<58; ++i) { + buckets.add(new BucketId(i, randValue)); + } + randValue = randomized.nextLong(); + for (int i=minUsedBits; i<58; ++i) { + buckets.add(new BucketId(i, randValue)); + } + return Collections.unmodifiableList(buckets); + } + @Override + public void tearDown() throws Exception { + if (test != null) { + System.err.println("Verified " + test.getVerifiedTests() + " test results for test " + test.getName()); + test.recordTestResults(); + } + } + public void testSimple() { + test = new DistributionTestFactory("simple"); + List<BucketId> buckets = getTestBuckets(); + Integer nodes[] = { 6, 3, 4, 8, 8, 8, 8, 8, 8, 3 }; + for (int i=0; i<buckets.size(); ++i) { + BucketId bucket = buckets.get(i); + DistributionTestFactory.Test t = test.recordResult(bucket).assertNodeCount(1); + if (i < nodes.length) t.assertNodeUsed(nodes[i]); + } + } + public void testDown() throws Exception { + test = new DistributionTestFactory("down") + .setUpStates("u") + .setClusterState(new ClusterState( + "distributor:10 .4.s:m .5.s:m .6.s:d .7.s:d .8.s:r .9.s:r")); + for (BucketId bucket : getTestBuckets()) { + assertTrue(test.recordResult(bucket).assertNodeCount(1).getNodes().get(0) < 4); + } + } + + /** + * The java side runs unit tests first. Thus java side will generate the distribution tests that the C++ side will verify equality with. + * The tests serialized by the java side will be checked into version control, such that C++ side can test without java side. When one of the sides + * change, the failing side can be identified by checking if the serialized files are modified from what is checked into version control. + */ + private void writeDistributionTest(String name, String clusterState, String distributionConfig) throws IOException, ParseException, Distribution.TooFewBucketBitsInUseException, Distribution.NoDistributorsAvailableException { + writeFileAtomically("src/tests/distribution/testdata/java_" + name + ".cfg", distributionConfig); + writeFileAtomically("src/tests/distribution/testdata/java_" + name + ".state", clusterState); + StringWriter sw = new StringWriter(); + Distribution distribution = new Distribution("raw:" + distributionConfig); + ClusterState state = new ClusterState(clusterState); + long maxBucket = 1; + for (int distributionBits = 0; distributionBits <= 32; ++distributionBits) { + state.setDistributionBits(distributionBits); + RandomGen randomizer = new RandomGen(distributionBits); + for (int bucketIndex = 0; bucketIndex < 64; ++bucketIndex) { + if (bucketIndex >= maxBucket) break; + long bucketId = bucketIndex; + // Use random bucket if we dont test all + if (maxBucket > 64) { + bucketId = randomizer.nextLong(); + } + BucketId bucket = new BucketId(distributionBits, bucketId); + for (int redundancy = 1; redundancy <= distribution.getRedundancy(); ++redundancy) { + int distributorIndex = distribution.getIdealDistributorNode(state, bucket, "uim"); + sw.write(distributionBits + " " + bucket.withoutCountBits() + " " + redundancy + " " + distributorIndex + "\n"); + } + } + maxBucket = maxBucket << 1; + } + + writeFileAtomically("src/tests/distribution/testdata/java_" + name + ".distribution", sw.toString()); + } + + private void writeFileAtomically(String filename, String data) throws IOException { + FileSystem fs = FileSystems.getDefault(); + Path filePath = fs.getPath(filename); + Path tempFilePath = fs.getPath(filename + ".tmp"); + + try (BufferedWriter bw = Files.newBufferedWriter(tempFilePath, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + bw.write(data); + } + + // Try to atomically move temporary file onto file. This is guaranteed to be atomic due to the files existing in the same file system + Files.move(tempFilePath, filePath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } + + + public void testWriteDistribution() throws IOException, ParseException, Distribution.TooFewBucketBitsInUseException, Distribution.NoDistributorsAvailableException { + String clusterState = "distributor:9"; + String distributionConfig = + "redundancy 3\n" + + "group[4]\n" + + "group[0].index \"invalid\"\n" + + "group[0].name \"invalid\"\n" + + "group[0].partitions 1|2|*\n" + + "group[0].nodes[0]\n" + + "group[1].index 1\n" + + "group[1].capacity 2.0\n" + + "group[1].name group1\n" + + "group[1].partitions *\n" + + "group[1].nodes[3]\n" + + "group[1].nodes[0].index 0\n" + + "group[1].nodes[1].index 1\n" + + "group[1].nodes[2].index 2\n" + + "group[2].index 2\n" + + "group[2].capacity 3.0\n" + + "group[2].name group2\n" + + "group[2].partitions *\n" + + "group[2].nodes[3]\n" + + "group[2].nodes[0].index 3\n" + + "group[2].nodes[1].index 4\n" + + "group[2].nodes[2].index 5\n" + + "group[3].index 3\n" + + "group[3].capacity 5.0\n" + + "group[3].name group3\n" + + "group[3].partitions *\n" + + "group[3].nodes[3]\n" + + "group[3].nodes[0].index 6\n" + + "group[3].nodes[1].index 7\n" + + "group[3].nodes[2].index 8\n"; + writeDistributionTest("depth2", clusterState, distributionConfig); + + clusterState = "distributor:20 storage:20"; + String complexDistributionConfig = + "redundancy 5\n" + + "group[7]\n" + + "group[0].partitions \"*|*\"\n" + + "group[0].index \"invalid\"\n" + + "group[0].name \"invalid\"\n" + + "group[0].nodes[0]\n" + + "group[1].partitions \"1|*\"\n" + + "group[1].index \"0\"\n" + + "group[1].name \"switch0\"\n" + + "group[1].nodes[0]\n" + + "group[2].partitions \"\"\n" + + "group[2].index \"0.0\"\n" + + "group[2].name \"rack0\"\n" + + "group[2].nodes[4]\n" + + "group[2].nodes[0].index 0\n" + + "group[2].nodes[1].index 1\n" + + "group[2].nodes[2].index 2\n" + + "group[2].nodes[3].index 3\n" + + "group[3].partitions \"\"\n" + + "group[3].index \"0.1\"\n" + + "group[3].name \"rack1\"\n" + + "group[3].nodes[4]\n" + + "group[3].nodes[0].index 8\n" + + "group[3].nodes[1].index 9\n" + + "group[3].nodes[2].index 14\n" + + "group[3].nodes[3].index 15\n" + + "group[4].partitions \"*\"\n" + + "group[4].index \"1\"\n" + + "group[4].name \"switch1\"\n" + + "group[4].nodes[0]\n" + + "group[5].partitions \"\"\n" + + "group[5].index \"1.0\"\n" + + "group[5].name \"rack0\"\n" + + "group[5].nodes[4]\n" + + "group[5].nodes[0].index 4\n" + + "group[5].nodes[1].index 5\n" + + "group[5].nodes[2].index 6\n" + + "group[5].nodes[3].index 17\n" + + "group[6].partitions \"\"\n" + + "group[6].index \"1.1\"\n" + + "group[6].name \"rack1\"\n" + + "group[6].nodes[4]\n" + + "group[6].nodes[0].index 10\n" + + "group[6].nodes[1].index 12\n" + + "group[6].nodes[2].index 13\n" + + "group[6].nodes[3].index 7"; + writeDistributionTest("depth3", clusterState, complexDistributionConfig); + + clusterState = "distributor:20 storage:20 .3.c:3 .7.c:2.5 .12.c:1.5"; + writeDistributionTest("capacity", clusterState, complexDistributionConfig); + + clusterState = "distributor:20 storage:20 .3.r:2 .7.r:3 .12.r:5"; + writeDistributionTest("retired", clusterState, complexDistributionConfig); + } + + public void testSplitBeyondSplitBitDoesntAffectDistribution() throws Exception { + Random randomized = new Random(7123161); + long val = randomized.nextLong(); + test = new DistributionTestFactory("abovesplitbit"); + for (int i=16; i<=58; ++i) { + test.recordResult(new BucketId(i, val)).assertNodeUsed(2); + } + } + public void testMinimalMovement() throws Exception { + test = new DistributionTestFactory("minimal-movement") + .setClusterState(new ClusterState("distributor:4 .2.s:d")); + DistributionTestFactory control = new DistributionTestFactory("minimal-movement-reference") + .setClusterState(new ClusterState("distributor:4")); + int moved = 0; + int staying = 0; + for (BucketId bucket : getTestBuckets()) { + DistributionTestFactory.Test org = control.recordResult(bucket).assertNodeCount(1); + DistributionTestFactory.Test res = test.recordResult(bucket).assertNodeCount(1); + if (org.getNodes().get(0) == 2) { + assertTrue(res.getNodes().get(0) != 2); + ++moved; + } else { + assertEquals(org, res); + ++staying; + } + } + assertEquals(63, moved); + assertEquals(81, staying); + } + public void testAllDistributionBits() throws Exception { + for (int distbits=0; distbits<=32; ++distbits) { + test = new DistributionTestFactory("distbit" + distbits) + .setClusterState(new ClusterState("bits:" + distbits + " distributor:10")); + List<BucketId> buckets = new ArrayList<>(); + for (int i=0; i<100; ++i) { + buckets.add(new BucketId(distbits, i)); + } + for (BucketId bucket : buckets) { + DistributionTestFactory.Test t = test.recordResult(bucket).assertNodeCount(1); + } + test.recordTestResults(); + test = null; + } + } + + private int getNodeCount(int depth, int branchCount, int nodesPerLeaf) { + if (depth <= 1) return branchCount * nodesPerLeaf; + int count = 0; + for (int i=0; i<branchCount; ++i) { + count += getNodeCount(depth - 1, branchCount, nodesPerLeaf); + } + return count; + } + private StorDistributionConfig.Builder buildHierarchicalConfig( + int redundancy, int branchCount, int depth, String partitions, int nodesPerLeaf) + { + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder() + .redundancy(redundancy); + builder.group(new StorDistributionConfig.Group.Builder() + .name("invalid").index("invalid").partitions(partitions)); + Stack<Integer> nodeIndexes = new Stack<>(); + for (int i=0, n=getNodeCount(depth, branchCount, nodesPerLeaf); i<n; ++i) nodeIndexes.push(i); + Collections.shuffle(nodeIndexes, new Random(123)); + addLevel(builder, "top", "", branchCount, depth, partitions, nodesPerLeaf, nodeIndexes); + return builder; + } + private void addLevel(StorDistributionConfig.Builder builder, String namePrefix, String indexPrefix, + int branchCount, int depth, String partitions, int nodesPerLeaf, + Stack<Integer> nodes) + { + for (int i=0; i<branchCount; ++i) { + StorDistributionConfig.Group.Builder group + = new StorDistributionConfig.Group.Builder(); + String gname = namePrefix + "." + i; + String index = (indexPrefix.isEmpty() ? "" + i : indexPrefix + "." + i); + group.name(gname).index(index); + if (depth <= 1) { + for (int j=0; j<nodesPerLeaf; ++j) { + group.nodes(new StorDistributionConfig.Group.Nodes.Builder().index(nodes.pop())); + } + } else { + group.partitions(partitions); + } + builder.group(group); + if (depth > 1) { + addLevel(builder, gname, index, branchCount, depth - 1, partitions, nodesPerLeaf, nodes); + } + } + } + + public void testHierarchicalDistribution() throws Exception { + test = new DistributionTestFactory("hierarchical-grouping") + .setDistribution(buildHierarchicalConfig(6, 3, 1, "1|2|*", 3)); + for (BucketId bucket : getTestBuckets()) { + test.recordResult(bucket).assertNodeCount(1); + } + } + public void testDistributorGroupTakeover() throws Exception { + test = new DistributionTestFactory("hierarchical-grouping-distributor-takeover") + .setDistribution(buildHierarchicalConfig(6, 3, 1, "1|2|*", 3).distributor_auto_ownership_transfer_on_whole_group_down(true)) + .setNodeType(NodeType.DISTRIBUTOR) + .setClusterState(new ClusterState("distributor:2 storage:9")); + for (BucketId bucket : getTestBuckets()) { + test.recordResult(bucket).assertNodeCount(1); + } + } + public void testDistributorNoGroupTakeover() throws Exception { + test = new DistributionTestFactory("hierarchical-grouping-distributor-notakeover") + .setDistribution(buildHierarchicalConfig(6, 3, 1, "1|2|*", 3).distributor_auto_ownership_transfer_on_whole_group_down(false)) + .setNodeType(NodeType.DISTRIBUTOR) + .setClusterState(new ClusterState("distributor:2 storage:9")); + int counts[] = new int[10]; + int noneExisting = 0; + for (BucketId bucket : getTestBuckets()) { + DistributionTestFactory.Test t = test.recordResult(bucket); + List<Integer> nodes = t.getNodes(); + if (nodes.isEmpty()) { + ++noneExisting; + t.assertFailure(DistributionTestFactory.Failure.NO_DISTRIBUTORS_AVAILABLE); + } else { + t.assertNodeCount(1); + for (int i : nodes) { + ++counts[i]; + } + } + } + for (int i=2; i<10; ++i) { + assertEquals(0, counts[i]); + } + for (int i=0; i<2; ++i) { + assertTrue(counts[i] > 0); + } + assertEquals(15, noneExisting); + } + public void testHierarchicalDistributionDeep() throws Exception { + System.out.println(new StorDistributionConfig(buildHierarchicalConfig(8, 5, 3, "*|*", 3))); + test = new DistributionTestFactory("hierarchical-grouping-deep") + .setNodeCount(500) + .setDistribution(buildHierarchicalConfig(8, 5, 3, "*|*", 3)); + for (BucketId bucket : getTestBuckets()) { + test.recordResult(bucket).assertNodeCount(1); + } + Set<ConfiguredNode> nodes = test.getDistribution().getNodes(); + // Despite setNodeCount(500) above, the actual distribution config + // itself only has 375 actual leaf nodes. + assertEquals(375, nodes.size()); + } + public void testHierarchicalDistributionCapacity() throws Exception { + StorDistributionConfig.Builder config = buildHierarchicalConfig(6, 3, 1, "1|*", 3); + config.group.get(1).capacity(3); + test = new DistributionTestFactory("group-capacity") + .setNodeCount(getNodeCount(1, 3, 3)).setDistribution(config); + + int counts[] = new int[9]; + for (int i=0; i<900; ++i) { + BucketId bucket = new BucketId(16, i); + ++counts[ test.recordResult(bucket).assertNodeCount(1).getNodes().get(0) ]; + } + int groupCount = 0; + for (StorDistributionConfig.Group.Nodes.Builder n : config.group.get(1).nodes) { + StorDistributionConfig.Group.Nodes node = new StorDistributionConfig.Group.Nodes(n); + groupCount += counts[ node.index() ]; + } + int avg3 = groupCount / 3; + int avg1 = (900 - groupCount) / 6; + double diff = 1.0 * avg3 / avg1; + assertTrue(Arrays.toString(counts) + ": Too large diff" + diff, diff < 3.1); + assertTrue(Arrays.toString(counts) + ": Too small diff" + diff, diff > 2.9); + } +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java new file mode 100644 index 00000000000..abd6ef70d35 --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java @@ -0,0 +1,250 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib.distribution; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.document.BucketId; +import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.vdslib.state.Node; +import com.yahoo.vdslib.state.NodeState; +import com.yahoo.vdslib.state.NodeType; +import com.yahoo.vespa.config.content.StorDistributionConfig; +import junit.framework.TestCase; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; + +import java.util.ArrayList; +import java.util.List; + +public class DistributionTestFactory extends CrossPlatformTestFactory { + private static final String testDirectory = "src/tests/distribution/testdata"; + private int redundancy; + private int nodeCount; + private ClusterState state; + private StorDistributionConfig.Builder distributionConfig; + private NodeType nodeType; + private String upStates; + + private int testsRecorded = 0; + private List<Test> results = new ArrayList<>(); + private int testsVerified = 0; + + enum Failure { NONE, TOO_FEW_BITS, NO_DISTRIBUTORS_AVAILABLE }; + + static public class Test { + private BucketId bucket; + private List<Integer> nodes; + private List<Integer> disks; + private Failure failure; + + public Test(BucketId bucket) { + this.bucket = bucket; + nodes = new ArrayList<>(); + disks = new ArrayList<>(); + failure = Failure.NONE; + } + + public boolean equals(Object other) { + if (!(other instanceof Test)) return false; + Test t = (Test) other; + return (bucket.equals(t.bucket) + && nodes.equals(t.nodes) + && failure.equals(t.failure)); + } + + public String toString() { + StringBuilder sb = new StringBuilder().append(bucket.toString()); + if (failure == Failure.NONE) { + sb.append(" ["); + for (int i=0; i<nodes.size(); ++i) { + if (i != 0) sb.append(" "); + sb.append(nodes.get(i)); + } + sb.append("]"); + } else { + sb.append(' ').append(failure); + } + return sb.toString(); + } + + public List<Integer> getNodes() { + return nodes; + } + public List<Integer> getDisks() { + return disks; + } + public Integer getDiskForNode(int node) { + for (int i=0; i<nodes.size(); ++i) { + if (nodes.get(i) == node) return disks.get(i); + } + TestCase.fail("Node " + node + " is not in use: " + toString()); + throw new IllegalStateException("Control should not reach here"); + } + + public Test assertFailure(Failure f) { + TestCase.assertEquals(f, failure); + return this; + } + public Test assertNodeCount(int count) { + if (count > 0) TestCase.assertEquals(toString(), Failure.NONE, failure); + TestCase.assertEquals(toString(), count, nodes.size()); + return this; + } + public Test assertNodeUsed(int node) { + TestCase.assertEquals(toString(), Failure.NONE, failure); + TestCase.assertTrue(toString(), nodes.contains(node)); + return this; + } + } + + public DistributionTestFactory(String name) { + super(testDirectory, name); + try{ + redundancy = 3; + nodeCount = 10; + state = new ClusterState("distributor:" + nodeCount); + distributionConfig = deserializeConfig(Distribution.getDefaultDistributionConfig(redundancy, nodeCount)); + nodeType = NodeType.DISTRIBUTOR; + upStates = "uim"; + loadTestResults(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static StorDistributionConfig.Builder deserializeConfig(String s) { + return new StorDistributionConfig.Builder( + new ConfigGetter<>(StorDistributionConfig.class).getConfig("raw:" + s)); + } + + public DistributionTestFactory setNodeCount(int count) throws Exception { + nodeCount = count; + distributionConfig = deserializeConfig(Distribution.getDefaultDistributionConfig(redundancy, nodeCount)); + state = new ClusterState("distributor:" + nodeCount); + return this; + } + + public DistributionTestFactory setClusterState(ClusterState state) { + this.state = state; + return this; + } + + public DistributionTestFactory setDistribution(StorDistributionConfig.Builder d) { + this.distributionConfig = d; + return this; + } + + public Distribution getDistribution() { + return new Distribution(new StorDistributionConfig(distributionConfig)); + } + + public DistributionTestFactory setNodeType(NodeType n) { + this.nodeType = n; + return this; + } + + public DistributionTestFactory setUpStates(String up) { + this.upStates = up; + return this; + } + + public int getVerifiedTests() { + return testsVerified; + } + + void verifySame(Test javaTest, Test other) { + TestCase.assertEquals("Reference test " + testsRecorded + " differ.", other, javaTest); + ++testsVerified; + } + + Test recordResult(BucketId bucket) { + Test t = new Test(bucket); + Distribution d = new Distribution(new StorDistributionConfig(distributionConfig)); + try{ + if (nodeType.equals(NodeType.DISTRIBUTOR)) { + int node = d.getIdealDistributorNode(state, bucket, upStates); + t.nodes.add(node); + } else { + for (int i : d.getIdealStorageNodes(state, bucket, upStates)) { + t.nodes.add(i); + NodeState ns = state.getNodeState(new Node(nodeType, i)); + if (ns.getDiskCount() != 0) { + t.disks.add(d.getIdealDisk(ns, i, bucket)); + } else { + t.disks.add(-1); + } + } + } + } catch (Distribution.TooFewBucketBitsInUseException e) { + t.failure = Failure.TOO_FEW_BITS; + } catch (Distribution.NoDistributorsAvailableException e) { + t.failure = Failure.NO_DISTRIBUTORS_AVAILABLE; + } + if (results.size() > testsRecorded) { + verifySame(t, results.get(testsRecorded)); + } else { + results.add(t); + } + ++testsRecorded; + return t; + } + + public String serialize() throws Exception { + JSONObject test = new JSONObject() + .put("cluster-state", state.toString()) + .put("distribution", new StorDistributionConfig(distributionConfig).toString()) + .put("node-type", nodeType.toString()) + .put("redundancy", redundancy) + .put("node-count", nodeCount) + .put("up-states", upStates); + JSONArray results = new JSONArray(); + for(Test t : this.results) { + JSONArray nodes = new JSONArray(); + for (int i : t.nodes) { + nodes.put(i); + } + JSONArray disks = new JSONArray(); + for (int i : t.disks) { + nodes.put(i); + } + JSONObject testResult = new JSONObject() + .put("bucket", Long.toHexString(t.bucket.getId())) + .put("nodes", nodes) + .put("failure", t.failure.toString()); + if (nodeType == NodeType.STORAGE) { + testResult.put("disks", disks); + } + results.put(testResult); + } + test.put("result", results); + return test.toString(2); + } + + public void parse(String serialized) throws Exception { + JSONObject json = new JSONObject(serialized); + upStates = json.getString("up-states"); + nodeCount = json.getInt("redundancy"); + redundancy = json.getInt("redundancy"); + state = new ClusterState(json.getString("cluster-state")); + distributionConfig = deserializeConfig(json.getString("distribution")); + nodeType = NodeType.get(json.getString("node-type")); + JSONArray results = json.getJSONArray("result"); + for (int i=0; i<results.length(); ++i) { + JSONObject result = results.getJSONObject(i); + Test t = new Test(new BucketId(Long.parseLong(result.getString("bucket"), 16))); + { + JSONArray nodes = result.getJSONArray("nodes"); + for (int j=0; j<nodes.length(); ++j) { + t.nodes.add(nodes.getInt(j)); + } + } + if (nodeType == NodeType.STORAGE) { + JSONArray disks = result.getJSONArray("disks"); + for (int j=0; j<disks.length(); ++j) { + t.disks.add(disks.getInt(j)); + } + } + t.failure = Failure.valueOf(result.getString("failure")); + this.results.add(t); + } + } +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/GroupTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/GroupTestCase.java new file mode 100644 index 00000000000..9d65b33e975 --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/GroupTestCase.java @@ -0,0 +1,188 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib.distribution; + +import java.text.ParseException; +import java.util.*; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +public class GroupTestCase extends junit.framework.TestCase { + + private void assertDistribution(String spec, int redundancy, String expectedResult) throws ParseException { + Group.Distribution distribution = new Group.Distribution(spec, redundancy); + assertEquals(spec, distribution.toString()); + int[] resultArray = distribution.getRedundancyArray(redundancy); + StringBuffer sb = new StringBuffer(); + for (int i=0; i<resultArray.length; ++i) { + if (i != 0) sb.append(','); + sb.append(resultArray[i]); + } + assertEquals("Spec: \"" + spec + "\", redundancy " + redundancy + " got unexpected result", expectedResult, sb.toString()); + } + + private void assertDistributionFailure(String spec, int redundancy, String expectedError) { + try{ + Group.Distribution distribution = new Group.Distribution(spec, redundancy); + assertTrue("Failed to fail parsing of spec \"" + spec + "\", redundancy " + redundancy + " with failure: " + expectedError, false); + } catch (Exception e) { + assertEquals(expectedError, e.getMessage()); + } + } + + public void testStarConversion() throws ParseException { + assertDistribution("1|*|*", 5, "2,2,1"); + assertDistribution("1|*|*", 6, "3,2,1"); + assertDistribution("1|*|*", 3, "1,1,1"); + assertDistribution("1|*|*", 2, "1,1"); + assertDistribution("4|*", 3, "3"); + assertDistribution("2|*", 3, "2,1"); + assertDistribution("*|*", 3, "2,1"); + assertDistribution("*|*|*", 4, "2,1,1"); + assertDistribution("*|*|*", 5, "2,2,1"); + assertDistribution("*|*|*|*", 5, "2,1,1,1"); + + assertDistributionFailure("2|*", 0, "The max redundancy must be a positive number in the range 1-255."); + assertDistributionFailure("*|2", 3, "Illegal distribution spec \"*|2\". Asterix specification must be tailing the specification."); + assertDistributionFailure("*|2|*", 3, "Illegal distribution spec \"*|2|*\". Asterix specification must be tailing the specification."); + assertDistributionFailure("0|*", 3, "Illegal distribution spec \"0|*\". Copy counts must be in the range 1-255."); + assertDistributionFailure("1|0|*", 3, "Illegal distribution spec \"1|0|*\". Copy counts must be in the range 1-255."); + assertDistributionFailure("1|a|*", 3, "Illegal distribution spec \"1|a|*\". Copy counts must be integer values in the range 1-255."); + assertDistributionFailure("1|999|*", 3, "Illegal distribution spec \"1|999|*\". Copy counts must be in the range 1-255."); + } + + private void setNodes(Group g, String nodes) { + StringTokenizer st = new StringTokenizer(nodes, ","); + List<ConfiguredNode> nodeList = new ArrayList<>(); + while (st.hasMoreTokens()) { + nodeList.add(new ConfiguredNode(Integer.parseInt(st.nextToken()), false)); + } + g.setNodes(nodeList); + } + + private void verifyUniqueHashes(Group g, Set<Integer> hashes) { + assertFalse(g.getName(), hashes.contains(g.getDistributionHash())); + hashes.add(g.getDistributionHash()); + } + + private Group buildGroupTree() throws ParseException { + Group root = new Group(5, "myroot", new Group.Distribution("2|*", 7)); + List<Group> level_one = new ArrayList<Group>(); + level_one.add(new Group(0, "br0", new Group.Distribution("1|1|*", 7))); + level_one.add(new Group(1, "br1", new Group.Distribution("*", 7))); + level_one.add(new Group(3, "br3", new Group.Distribution("8|*", 7))); + level_one.add(new Group(4, "br4", new Group.Distribution("1|*", 7))); + level_one.add(new Group(5, "br5", new Group.Distribution("*|*|*", 7))); + level_one.add(new Group(6, "br6", new Group.Distribution("*|*|*|*|*|*", 7))); + level_one.add(new Group(7, "br7", new Group.Distribution("*", 7))); + level_one.add(new Group(9, "br9", new Group.Distribution("1|*", 7))); + for(Group g : level_one) { + root.addSubGroup(g); + for (int i=0; i<5; ++i) { + Group child = new Group(i, g.getName() + "." + i); + g.addSubGroup(child); + List<ConfiguredNode> nodeList = new ArrayList<>(); + for (int j=0; j<5; ++j) nodeList.add(new ConfiguredNode(g.getIndex() * 10 + j, false)); + child.setNodes(nodeList); + } + } + // Create some irregularities + setNodes(level_one.get(3).getSubgroups().get(2), "19,7,8,17"); + try{ + Group br8 = new Group(5, "br8", new Group.Distribution("*", 5)); + root.addSubGroup(br8); + assertTrue(false); // Should fail index 5 is in use at that level + } catch (Exception e) { + assertEquals("A subgroup with index 5 already exist.", e.getMessage()); + } + try{ + Group br8 = new Group(5, "br8"); + Group br9 = new Group(2, "br9"); + br8.addSubGroup(br9); + assertTrue(false); // Should fail as we want distribution to be set on non-leaf node + } catch (Exception e) { + assertEquals("Cannot add sub groups to a node without distribution set.", e.getMessage()); + } + try{ + Group br8 = new Group(5, "br8", new Group.Distribution("*", 5)); + setNodes(br8, "1,2,3"); + assertTrue(false); // Should fail as we can't have distribution on leaf node. + } catch (Exception e) { + assertEquals("Cannot add nodes to non-leaf group with distribution set", e.getMessage()); + } + root.calculateDistributionHashValues(); + Set<Integer> distributionHashes = new HashSet<Integer>(); + verifyUniqueHashes(root, distributionHashes); + return root; + } + + public void testNormalusage() throws ParseException { + Group root = new Group(2, "myroot", new Group.Distribution("*", 2)); + assertFalse(root.isLeafGroup()); + + Group branch = new Group(5, "myleaf"); + assertTrue(branch.isLeafGroup()); + + root = buildGroupTree(); + + String expected = "Group(name: myroot, index: 5, distribution: 2|*, subgroups: 8) {\n" + + " Group(name: br0, index: 0, distribution: 1|1|*, subgroups: 5) {\n" + + " Group(name: br0.0, index: 0, nodes( 0 1 2 3 4 )) {\n" + + " }\n" + + " Group(name: br0.1, index: 1, nodes( 0 1 2 3 4 )) {\n" + + " }\n" + + " Group(name: br0.2, index: 2, nodes( 0 1 2 3 4 )) {\n" + + " }\n" + + " Group(name: br0.3, index: 3, nodes( 0 1 2 3 4 )) {\n" + + " }\n" + + " Group(name: br0.4, index: 4, nodes( 0 1 2 3 4 )) {\n" + + " }\n" + + " }\n"; + assertEquals(expected, root.toString().substring(0, expected.length())); + + assertEquals("br5.br5.0", root.getGroupForNode(52).getPath()); + } + + private Group.Distribution dummyDistribution() throws Exception { + return new Group.Distribution("*", 1); + } + + public void testRootGroupDoesNotIncludeGroupNameWhenNoChildren() { + Group g = new Group(0, "donkeykong"); + assertThat(g.getUnixStylePath(), is("/")); + } + + public void testChildNamesDoNotIncludeRootGroupName() throws Exception { + Group g = new Group(0, "donkeykong", dummyDistribution()); + Group child = new Group(1, "mario"); + g.addSubGroup(child); + assertThat(child.getUnixStylePath(), is("/mario")); + } + + public void testNestedGroupsAreSlashSeparated() throws Exception { + Group g = new Group(0, "donkeykong", dummyDistribution()); + Group mario = new Group(1, "mario", dummyDistribution()); + Group toad = new Group(2, "toad"); + mario.addSubGroup(toad); + g.addSubGroup(mario); + + assertThat(toad.getUnixStylePath(), is("/mario/toad")); + } + + public void testMultipleLeafGroupsAreEnumerated() throws Exception { + Group g = new Group(0, "donkeykong", dummyDistribution()); + Group mario = new Group(1, "mario", dummyDistribution()); + Group toad = new Group(2, "toad"); + mario.addSubGroup(toad); + Group yoshi = new Group(3, "yoshi"); + mario.addSubGroup(yoshi); + g.addSubGroup(mario); + Group luigi = new Group(4, "luigi"); + g.addSubGroup(luigi); + + assertThat(toad.getUnixStylePath(), is("/mario/toad")); + assertThat(yoshi.getUnixStylePath(), is("/mario/yoshi")); + assertThat(luigi.getUnixStylePath(), is("/luigi")); + } + +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java new file mode 100644 index 00000000000..c058a7c9919 --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java @@ -0,0 +1,246 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib.state; + +import java.text.ParseException; + +public class ClusterStateTestCase extends junit.framework.TestCase { + + public void testSetNodeState() throws ParseException { + ClusterState state = new ClusterState(""); + assertEquals("", state.toString()); + state.setNodeState(new Node(NodeType.DISTRIBUTOR, 3), new NodeState(NodeType.DISTRIBUTOR, State.UP)); + assertEquals("distributor:4 .0.s:d .1.s:d .2.s:d", state.toString()); + state.setNodeState(new Node(NodeType.DISTRIBUTOR, 1), new NodeState(NodeType.DISTRIBUTOR, State.UP)); + assertEquals("distributor:4 .0.s:d .2.s:d", state.toString()); + state.setNodeState(new Node(NodeType.DISTRIBUTOR, 3), new NodeState(NodeType.DISTRIBUTOR, State.DOWN)); + assertEquals("distributor:2 .0.s:d", state.toString()); + state.setNodeState(new Node(NodeType.DISTRIBUTOR, 4), new NodeState(NodeType.DISTRIBUTOR, State.UP)); + assertEquals("distributor:5 .0.s:d .2.s:d .3.s:d", state.toString()); + state.setNodeState(new Node(NodeType.STORAGE, 0), new NodeState(NodeType.STORAGE, State.UP).setDiskCount(4)); + assertEquals("distributor:5 .0.s:d .2.s:d .3.s:d storage:1", state.toString()); + state.setNodeState(new Node(NodeType.STORAGE, 0), new NodeState(NodeType.STORAGE, State.UP).setDiskCount(4).setDiskState(1, new DiskState(State.DOWN))); + assertEquals("distributor:5 .0.s:d .2.s:d .3.s:d storage:1 .0.d:4 .0.d.1.s:d", state.toString()); + } + + public void testClone() throws ParseException { + ClusterState state = new ClusterState(""); + state.setNodeState(new Node(NodeType.DISTRIBUTOR, 1), new NodeState(NodeType.DISTRIBUTOR, State.UP).setDescription("available")); + state.setNodeState(new Node(NodeType.STORAGE, 0), new NodeState(NodeType.STORAGE, State.UP).setCapacity(1.2).setReliability(2)); + state.setNodeState(new Node(NodeType.STORAGE, 2), new NodeState(NodeType.STORAGE, State.UP).setDiskCount(2).setDiskState(1, new DiskState(State.DOWN))); + ClusterState other = state.clone(); + assertEquals(state.toString(true), other.toString(true)); + assertEquals(state.toString(false), other.toString(false)); + assertEquals(state, other); + } + + public void testEquals() throws ParseException { + ClusterState state = new ClusterState(""); + + assertEquals(state, new ClusterState("")); + + assertEquals(state, new ClusterState("version:0")); + assertEquals(state, new ClusterState("cluster:u")); + assertEquals(state, new ClusterState("bits:16")); + assertEquals(state, new ClusterState("storage:0")); + assertEquals(state, new ClusterState("distributor:0")); + + assertFalse(state.equals(new ClusterState("version:1"))); + assertFalse(state.equals(new ClusterState("cluster:d"))); + assertFalse(state.equals(new ClusterState("bits:20"))); + assertFalse(state.equals(new ClusterState("storage:1"))); + assertFalse(state.equals(new ClusterState("distributor:1"))); + + { + ClusterState state1 = new ClusterState("distributor:3 .1.s:d .2.s:m storage:3 .1.s:i .2.s:r"); + ClusterState state2 = new ClusterState("distributor:3 .1.s:d .2.s:m storage:3 .1.s:i .2.s:m"); + assertFalse(state1.equals(state2)); + assertFalse(state1.similarTo(state2)); + } + + { + ClusterState state1 = new ClusterState("cluster:d"); + ClusterState state2 = new ClusterState("cluster:d version:1 bits:20 distributor:1 storage:1 .0.s:d"); + assertFalse(state1.equals(state2)); + assertTrue(state1.similarTo(state2)); + } + + { + ClusterState state1 = new ClusterState("distributor:3 .1.s:d .2.s:m storage:3 .1.s:i .2.s:r"); + ClusterState state2 = new ClusterState("distributor:3 storage:3"); + assertFalse(state1.equals(state2)); + assertFalse(state1.similarTo(state2)); + } + + assertFalse(state.equals("class not instance of ClusterState")); + assertFalse(state.similarTo("class not instance of ClusterState")); + + assertEquals(state, state); + assertTrue(state.similarTo(state)); + } + + public void testTextDiff() throws ParseException { + ClusterState state1 = new ClusterState("distributor:9 storage:4"); + ClusterState state2 = new ClusterState("distributor:7 storage:6"); + ClusterState state3 = new ClusterState("distributor:9 storage:2"); + + assertEquals("storage: [4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2)); + assertEquals("storage: [2: Up => Down, 3: Up => Down, 4: Up => Down, 5: Up => Down], distributor: [7: Down => Up, 8: Down => Up]", state2.getTextualDifference(state3)); + + state2.setDistributionBits(21); + state1.setVersion(123); + state1.setNodeState(new Node(NodeType.STORAGE, 2), new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(0.2).setDiskCount(2).setDescription("Booting")); + state2.setOfficial(true); + + assertEquals("version: 123 => 0, bits: 16 => 21, official: false => true, storage: [2: [Initializing => Up, disks: 2 => 0, description: Booting => ], 4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2)); + } + + public void testHtmlDiff() throws ParseException { + ClusterState state1 = new ClusterState("distributor:9 storage:4"); + ClusterState state2 = new ClusterState("distributor:7 storage:6"); + ClusterState state3 = new ClusterState("distributor:9 storage:2"); + + assertEquals("storage: [4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2)); + assertEquals("storage: [<br>\n" + + " 4: <b>Down</b> => <b>Up</b>, <br>\n" + + " 5: <b>Down</b> => <b>Up</b><br>\n" + + "], distributor: [<br>\n" + + " 7: <b>Up</b> => <b>Down</b>, <br>\n" + + " 8: <b>Up</b> => <b>Down</b><br>\n" + + "]", state1.getHtmlDifference(state2)); + assertEquals("storage: [2: Up => Down, 3: Up => Down, 4: Up => Down, 5: Up => Down], distributor: [7: Down => Up, 8: Down => Up]", state2.getTextualDifference(state3)); + assertEquals("storage: [<br>\n" + + " 2: <b>Up</b> => <b>Down</b>, <br>\n" + + " 3: <b>Up</b> => <b>Down</b>, <br>\n" + + " 4: <b>Up</b> => <b>Down</b>, <br>\n" + + " 5: <b>Up</b> => <b>Down</b><br>\n" + + "], distributor: [<br>\n" + + " 7: <b>Down</b> => <b>Up</b>, <br>\n" + + " 8: <b>Down</b> => <b>Up</b><br>\n" + + "]", state2.getHtmlDifference(state3)); + + state1.setVersion(123); + state1.setNodeState(new Node(NodeType.STORAGE, 2), new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(0.2).setDiskCount(2).setDescription("Booting")); + state2.setDistributionBits(21); + state2.setOfficial(true); + assertEquals("version: 123 => 0, bits: 16 => 21, official: false => true, storage: [2: [Initializing => Up, disks: 2 => 0, description: Booting => ], 4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2)); + assertEquals("version: 123 => 0, bits: 16 => 21, official: false => true, storage: [<br>\n" + + " 2: [<b>Initializing</b> => <b>Up</b>, disks: 2 => 0, description: Booting => ], <br>\n" + + " 4: <b>Down</b> => <b>Up</b>, <br>\n" + + " 5: <b>Down</b> => <b>Up</b><br>\n" + + "], distributor: [<br>\n" + + " 7: <b>Up</b> => <b>Down</b>, <br>\n" + + " 8: <b>Up</b> => <b>Down</b><br>\n" + + "]", state1.getHtmlDifference(state2)); + } + + + public void testParser() throws ParseException { + ClusterState state = new ClusterState("distributor:2 storage:17 .2.s:d .13.s:r m:cluster\\x20message"); + assertEquals("cluster message", state.getDescription()); + + for (int i = 0; i < state.getNodeCount(NodeType.STORAGE); i++) { + if (i == 2) + assertEquals("d", state.getNodeState(new Node(NodeType.STORAGE, i)).getState().serialize()); + if (i == 13) + assertEquals("r", state.getNodeState(new Node(NodeType.STORAGE, i)).getState().serialize()); + if (i != 2 && i != 13) + assertEquals("u", state.getNodeState(new Node(NodeType.STORAGE, i)).getState().serialize()); + } + + assertEquals("distributor:2 storage:17 .2.s:d .13.s:r", state.toString()); + assertEquals("distributor:2", new ClusterState("distributor:2 storage:0").toString()); + assertEquals("storage:2", new ClusterState("storage:2 .0.d:3 .1.d:4").toString()); + assertEquals("distributor:2 .1.t:3 storage:2", new ClusterState("whatever:4 distributor:2 .1.t:3 storage:2").toString()); + assertEquals("distributor:2 storage:2", new ClusterState(": distributor:2 storage:2 .0:d cbadkey:u bbadkey:2 vbadkey:2 mbadkey:message dbadkey:5 sbadkey:6 unknownkey:somevalue").toString()); + + try { + new ClusterState("badtokenwithoutcolon"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + new ClusterState(".0.s:d"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + new ClusterState("cluster:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + new ClusterState("cluster:m"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + new ClusterState("version:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + new ClusterState("distributor:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + new ClusterState("storage:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + new ClusterState("distributor:2 .3.s:d"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + new ClusterState("storage:2 .3.s:d"); + assertTrue("Should fail", false); + } catch (Exception e) {} + } + + public void testCapacityExponential() throws ParseException { + + ClusterState state = new ClusterState("distributor:27 storage:170 .2.s:d .13.c:3E-8 .13.s:r"); + assertEquals(3E-8, state.getNodeState(new Node(NodeType.STORAGE, 13)).getCapacity()); + } + + public void testCapacityExponentialCpp() throws ParseException { + ClusterState state = new ClusterState("distributor:27 storage:170 .2.s:d .13.c:3e-08 .13.s:r"); + assertEquals(3E-8, state.getNodeState(new Node(NodeType.STORAGE, 13)).getCapacity()); + } + + public void testSetState() throws ParseException { + ClusterState state = new ClusterState("distributor:2 storage:2"); + state.setNodeState(new Node(NodeType.DISTRIBUTOR, 0), new NodeState(NodeType.DISTRIBUTOR, State.DOWN)); + + assertEquals("distributor:2 .0.s:d storage:2", state.toString()); + } + + public void testVersionAndClusterStates() throws ParseException { + ClusterState state = new ClusterState("version:4 cluster:i distributor:2 .1.s:i storage:2 .0.s:i .0.i:0.345"); + assertEquals(4, state.getVersion()); + assertEquals(State.INITIALIZING, state.getClusterState()); + assertEquals(0.345, state.getNodeState(new Node(NodeType.STORAGE, 0)).getInitProgress(), 0.000001); + state.setClusterState(State.DOWN); + state.setVersion(5); + state.setDistributionBits(12); + assertEquals("version:5 cluster:d bits:12 distributor:2 .1.s:i .1.i:1.0 storage:2 .0.s:i .0.i:0.345", state.toString()); + } + + public void testNotRemovingCommentedDownNodesAtEnd() throws ParseException { + ClusterState state = new ClusterState(""); + state.setNodeState(new Node(NodeType.DISTRIBUTOR, 0), new NodeState(NodeType.DISTRIBUTOR, State.UP)); + state.setNodeState(new Node(NodeType.DISTRIBUTOR, 1), new NodeState(NodeType.DISTRIBUTOR, State.UP)); + state.setNodeState(new Node(NodeType.STORAGE, 0), new NodeState(NodeType.STORAGE, State.UP)); + state.setNodeState(new Node(NodeType.STORAGE, 1), new NodeState(NodeType.STORAGE, State.UP)); + state.setNodeState(new Node(NodeType.STORAGE, 2), new NodeState(NodeType.STORAGE, State.UP)); + assertEquals("distributor:2 storage:3", state.toString()); + state.setNodeState(new Node(NodeType.STORAGE, 2), new NodeState(NodeType.STORAGE, State.DOWN).setDescription("Took it down")); + state.setNodeState(new Node(NodeType.DISTRIBUTOR, 1), new NodeState(NodeType.DISTRIBUTOR, State.DOWN).setDescription("Took it down")); + assertEquals("distributor:2 .1.s:d .1.m:Took\\x20it\\x20down storage:3 .2.s:d .2.m:Took\\x20it\\x20down", state.toString(true)); + assertEquals("distributor:1 storage:2", state.toString(false)); + } + + public void testWhitespace() throws ParseException { + ClusterState state = new ClusterState("distributor:2\n .1.t:3\nstorage:2\n\t.0.s:i \r\f.1.s:m"); + assertEquals(2, state.getNodeCount(NodeType.DISTRIBUTOR)); + assertEquals(2, state.getNodeCount(NodeType.STORAGE)); + assertEquals(new NodeState(NodeType.DISTRIBUTOR, State.UP), state.getNodeState(new Node(NodeType.DISTRIBUTOR, 0))); + assertEquals(new NodeState(NodeType.DISTRIBUTOR, State.UP).setStartTimestamp(3), state.getNodeState(new Node(NodeType.DISTRIBUTOR, 1))); + assertEquals(new NodeState(NodeType.STORAGE, State.INITIALIZING), state.getNodeState(new Node(NodeType.STORAGE, 0))); + assertEquals(new NodeState(NodeType.STORAGE, State.MAINTENANCE), state.getNodeState(new Node(NodeType.STORAGE, 1))); + } +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/state/DiskStateTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/state/DiskStateTestCase.java new file mode 100644 index 00000000000..9d78b54a681 --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/state/DiskStateTestCase.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib.state; + +import java.text.ParseException; + +public class DiskStateTestCase extends junit.framework.TestCase { + + public void testEquals() { + DiskState d1 = new DiskState(State.UP, "", 1); + DiskState d2 = new DiskState(State.UP, "", 2); + DiskState d3 = new DiskState(State.DOWN, "Failed disk", 1); + DiskState d4 = new DiskState(State.DOWN, "IO error", 1); + DiskState d5 = new DiskState(State.UP, "", 1); + DiskState d6 = new DiskState(State.UP, "", 2); + DiskState d7 = new DiskState(State.DOWN, "Failed disk", 1); + DiskState d8 = new DiskState(State.DOWN, "IO error", 1); + + assertEquals(d1, d5); + assertEquals(d2, d6); + assertEquals(d3, d7); + assertEquals(d4, d8); + + assertFalse(d1.equals(d2)); + assertFalse(d1.equals(d3)); + assertFalse(d1.equals(d4)); + + assertFalse(d2.equals(d1)); + assertFalse(d2.equals(d3)); + assertFalse(d2.equals(d4)); + + assertFalse(d3.equals(d1)); + assertFalse(d3.equals(d2)); + assertEquals(d3, d4); + + assertFalse(d4.equals(d1)); + assertFalse(d4.equals(d2)); + assertEquals(d4, d3); + + assertFalse(d1.equals("class not instance of Node")); + } + + public void testSerialization() throws ParseException { + DiskState d = new DiskState(); + DiskState other = new DiskState(d.serialize("", true)); + assertEquals(d, other); + assertEquals(d.toString(), other.toString()); + assertEquals(State.UP, other.getState()); + assertEquals(1.0, other.getCapacity()); + assertEquals("", other.getDescription()); + assertEquals("s:u", d.serialize("", false)); + assertEquals("s:u", d.serialize("", true)); + assertEquals("", d.serialize(".0.", false)); + assertEquals("", d.serialize(".0.", true)); + + assertEquals(d, new DiskState(": s:u sbadkey:somevalue cbadkey:somevalue mbadkey:somevalue unknwonkey:somevalue")); + + d = new DiskState(State.UP, "Slow disk", 1.0); + other = new DiskState(d.serialize("", true)); + assertEquals(d, other); + assertEquals(d.toString(), other.toString()); + assertEquals(State.UP, other.getState()); + assertEquals(1.0, other.getCapacity()); + assertEquals("Slow disk", other.getDescription()); + assertEquals("s:u", d.serialize("", false)); + assertEquals("s:u m:Slow\\x20disk", d.serialize("", true)); + assertEquals("", d.serialize(".0.", false)); + assertEquals(".0.m:Slow\\x20disk", d.serialize(".0.", true)); + + d = new DiskState(State.DOWN, "Failed disk", 2.0); + other = new DiskState(d.serialize("", true)); + assertEquals(d, other); + assertEquals(d.toString(), other.toString()); + assertEquals(State.DOWN, other.getState()); + assertEquals(2.0, other.getCapacity()); + assertEquals("Failed disk", other.getDescription()); + assertEquals("s:d c:2.0", d.serialize("", false)); + assertEquals("s:d c:2.0 m:Failed\\x20disk", d.serialize("", true)); + assertEquals(".0.s:d .0.c:2.0", d.serialize(".0.", false)); + assertEquals(".0.s:d .0.c:2.0 .0.m:Failed\\x20disk", d.serialize(".0.", true)); + + try { + new DiskState(State.MAINTENANCE); + assertTrue("Method expected to throw IllegalArgumentException", false); + } catch (IllegalArgumentException e) { + assertEquals("State " + State.MAINTENANCE + " is not a valid disk state.", e.getMessage()); + } + try { + new DiskState(State.UP, "", -1); + assertTrue("Method expected to throw IllegalArgumentException", false); + } catch (IllegalArgumentException e) { + assertEquals("Negative capacity makes no sense.", e.getMessage()); + } + try { + new DiskState("nocolon"); + assertTrue("Method expected to throw ParseException", false); + } catch (ParseException e) { + assertEquals("Token nocolon does not contain ':': nocolon", e.getMessage()); + } + try { + new DiskState("s:d c:badvalue"); + assertTrue("Method expected to throw ParseException", false); + } catch (ParseException e) { + assertEquals("Illegal disk capacity 'badvalue'. Capacity must be a positive floating point number", e.getMessage()); + } + } +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/state/NodeStateTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/state/NodeStateTestCase.java new file mode 100644 index 00000000000..63137a92c7b --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/state/NodeStateTestCase.java @@ -0,0 +1,231 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib.state; + +import java.text.ParseException; +import java.util.List; + +public class NodeStateTestCase extends junit.framework.TestCase { + + public void testOrdinals() { + NodeType nt = NodeType.STORAGE; + + // All states are above maintenance + assertFalse(new NodeState(nt, State.MAINTENANCE).above(new NodeState(nt, State.MAINTENANCE))); + assertTrue(new NodeState(nt, State.DOWN).above(new NodeState(nt, State.MAINTENANCE))); + assertTrue(new NodeState(nt, State.STOPPING).above(new NodeState(nt, State.MAINTENANCE))); + assertTrue(new NodeState(nt, State.INITIALIZING).above(new NodeState(nt, State.MAINTENANCE))); + assertTrue(new NodeState(nt, State.RETIRED).above(new NodeState(nt, State.MAINTENANCE))); + assertTrue(new NodeState(nt, State.UP).above(new NodeState(nt, State.MAINTENANCE))); + + // Most are above down + assertFalse(new NodeState(nt, State.MAINTENANCE).above(new NodeState(nt, State.DOWN))); + assertFalse(new NodeState(nt, State.DOWN).above(new NodeState(nt, State.DOWN))); + assertTrue(new NodeState(nt, State.STOPPING).above(new NodeState(nt, State.DOWN))); + assertTrue(new NodeState(nt, State.INITIALIZING).above(new NodeState(nt, State.DOWN))); + assertTrue(new NodeState(nt, State.RETIRED).above(new NodeState(nt, State.DOWN))); + assertTrue(new NodeState(nt, State.UP).above(new NodeState(nt, State.DOWN))); + + // Only up is above retired + assertFalse(new NodeState(nt, State.MAINTENANCE).above(new NodeState(nt, State.RETIRED))); + assertFalse(new NodeState(nt, State.DOWN).above(new NodeState(nt, State.RETIRED))); + assertFalse(new NodeState(nt, State.STOPPING).above(new NodeState(nt, State.RETIRED))); + assertFalse(new NodeState(nt, State.INITIALIZING).above(new NodeState(nt, State.RETIRED))); + assertFalse(new NodeState(nt, State.RETIRED).above(new NodeState(nt, State.RETIRED))); + assertTrue(new NodeState(nt, State.UP).above(new NodeState(nt, State.RETIRED))); + } + + public void testTrivialitiesToIncreaseCoverage() throws ParseException { + NodeState ns = new NodeState(NodeType.STORAGE, State.UP); + assertEquals(1, ns.getReliability()); + assertEquals(false, ns.isAnyDiskDown()); + + assertEquals(ns.setReliability(2).serialize(), NodeState.deserialize(NodeType.STORAGE, "r:2").serialize()); + assertEquals(ns.setDiskCount(1).serialize(), NodeState.deserialize(NodeType.STORAGE, "r:2 d:1").serialize()); + assertEquals(ns.setReliability(1).serialize(), NodeState.deserialize(NodeType.STORAGE, "d:1").serialize()); + + assertEquals(ns.setDiskState(0, new DiskState(State.DOWN, "", 1)), NodeState.deserialize(NodeType.STORAGE, "s:u d:1 d.0.s:d")); + assertEquals(ns, NodeState.deserialize(NodeType.STORAGE, "s:u d:1 d.0:d")); + } + + public void testDiskState() throws ParseException { + NodeState ns = NodeState.deserialize(NodeType.STORAGE, "s:m"); + assertEquals(new DiskState(State.UP, "", 1), ns.getDiskState(0)); + assertEquals(new DiskState(State.UP, "", 1), ns.getDiskState(1)); + assertEquals(new DiskState(State.UP, "", 1), ns.getDiskState(100)); + + ns.setDiskCount(2).setDiskState(1, new DiskState(State.DOWN, "bad disk", 1)); + assertEquals(new DiskState(State.UP, "", 1), ns.getDiskState(0)); + assertEquals(new DiskState(State.DOWN, "bad disk", 1), ns.getDiskState(1)); + + List<DiskState> diskStates = ns.getDiskStates(); + assertEquals(2, diskStates.size()); + for (int i=0; i<diskStates.size(); i++) { + DiskState diskState = diskStates.get(i); + if (i==1) { + assertEquals(new DiskState(State.DOWN, "bad disk", 1), diskState); + } else { + assertEquals(new DiskState(State.UP, "", 1), diskState); + } + } + + try { + NodeState.deserialize(NodeType.STORAGE, "s:m").setDiskCount(-1); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m").setDiskState(1, new DiskState(State.DOWN, "bad disk", 1)); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m").setDiskCount(2).setDiskState(1, new DiskState(State.DOWN, "bad disk", 1)).getDiskState(100); + assertTrue("Should fail", false); + } catch (Exception e) {} + } + + public void testSerialization() throws ParseException { + NodeState ns = new NodeState(NodeType.STORAGE, State.MAINTENANCE); + assertEquals("s:m", ns.serialize(false)); + assertEquals("s:m", ns.serialize(true)); + assertEquals(ns, NodeState.deserialize(NodeType.STORAGE, "s:m")); + assertEquals(ns, NodeState.deserialize(NodeType.STORAGE, "s:m c:1.0 r:1 d:0 t:0")); + + NodeState nsd = new NodeState(NodeType.DISTRIBUTOR, State.MAINTENANCE); + assertEquals(nsd, NodeState.deserialize(NodeType.DISTRIBUTOR, "s:m")); + assertEquals(nsd, NodeState.deserialize(NodeType.DISTRIBUTOR, "s:m c:2.0 r:2 d:2")); // Ignore capacity, reliability, and disk count for distributors + + assertEquals(ns, NodeState.deserialize(NodeType.STORAGE, ": s:m sbadkey:u bbadkey:2 cbadkey:2.0 rbadkey:2 ibadkey:0.5 tbadkey:2 mbadkey:message dbadkey:2 unknownkey:somevalue")); + try { + NodeState.deserialize(NodeType.STORAGE, "s:m badtokenwithoutcolon"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m c:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m r:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m i:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m t:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m t:-1"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m d:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m d.badkey:badvalue"); + assertTrue("Should fail", false); + } catch (Exception e) {} + try { + NodeState.deserialize(NodeType.STORAGE, "s:m d.1:badindex"); + assertTrue("Should fail", false); + } catch (Exception e) {} + + ns = new NodeState(NodeType.STORAGE, State.UP).setDescription("Foo bar"); + assertEquals("", ns.serialize(2, false)); + assertEquals("m:Foo\\x20bar", ns.serialize(false)); + assertEquals("m:Foo\\x20bar", ns.serialize(true)); + + ns = new NodeState(NodeType.STORAGE, State.MAINTENANCE).setDescription("Foo bar").setCapacity(1.2).setDiskCount(4) + .setMinUsedBits(12).setStartTimestamp(5).setDiskState(1, new DiskState(State.DOWN, "bad disk", 1)) + .setDiskState(3, new DiskState(State.UP, "", 2)); + assertEquals(".2.s:m .2.c:1.2 .2.t:5 .2.d:4 .2.d.1.s:d .2.d.3.c:2.0", ns.serialize(2, false)); + assertEquals("s:m c:1.2 t:5 b:12 d:4 d.1.s:d d.3.c:2.0 m:Foo\\x20bar", ns.serialize(false)); + assertEquals("s:m c:1.2 t:5 b:12 d:4 d.1.s:d d.1.m:bad\\x20disk d.3.c:2.0 m:Foo\\x20bar", ns.serialize(true)); + NodeState ns2 = NodeState.deserialize(NodeType.STORAGE, "s:m c:1.2 t:5 b:12 d:4 d.1.s:d d.1.m:bad\\x20disk d.3.c:2.0 m:Foo\\x20bar"); + assertEquals(ns, ns2); + + NodeState copy1 = NodeState.deserialize(NodeType.STORAGE, ns.serialize(false)); + NodeState copy2 = NodeState.deserialize(NodeType.STORAGE, ns.serialize(true)); + assertEquals(ns, copy1); + assertEquals(ns, copy2); + assertEquals(copy1, copy2); + assertEquals(ns.serialize(false), copy1.serialize(false)); + assertEquals(ns.serialize(false), copy2.serialize(false)); + assertEquals(ns.serialize(true), copy2.serialize(true)); + } + + public void testSimilarTo() { + { + NodeState ns1 = new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(0); + NodeState ns2 = new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(NodeState.getListingBucketsInitProgressLimit() / 2); + NodeState ns3 = new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(NodeState.getListingBucketsInitProgressLimit()); + NodeState ns4 = new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(NodeState.getListingBucketsInitProgressLimit() * 2); + assertTrue(ns1.similarTo(ns2)); + assertFalse(ns2.similarTo(ns3)); + assertTrue(ns3.similarTo(ns4)); + + assertFalse(ns1.equals(ns2)); + assertFalse(ns2.equals(ns3)); + assertFalse(ns3.equals(ns4)); + + assertFalse(ns1.equals("class not instance of NodeState")); + assertFalse(ns1.similarTo("class not instance of NodeState")); + } + { + NodeState ns1 = new NodeState(NodeType.STORAGE, State.UP).setMinUsedBits(16); + NodeState ns2 = new NodeState(NodeType.STORAGE, State.UP).setMinUsedBits(18); + assertTrue(ns1.similarTo(ns2)); + assertFalse(ns1.equals(ns2)); + } + { + NodeState ns = new NodeState(NodeType.STORAGE, State.MAINTENANCE); + NodeState ns2Disks = new NodeState(NodeType.STORAGE, State.MAINTENANCE).setDiskCount(2); + assertEquals(ns, ns2Disks); + assertEquals(ns2Disks, ns); + assertTrue(ns.similarTo(ns2Disks)); + assertTrue(ns2Disks.similarTo(ns)); + + ns2Disks.getDiskState(0).setState(State.DOWN); + assertFalse(ns.equals(ns2Disks)); + assertFalse(ns2Disks.equals(ns)); + assertFalse(ns.similarTo(ns2Disks)); + assertFalse(ns2Disks.similarTo(ns)); + } + } + + public void testReadableOutput() { + // toString() and getDiff() is mostly there just to make good error reports when unit tests fails. Make sure toString() is actually run with no test failures + // to make sure coverage doesn't complain when no test is failing. + NodeState ns = new NodeState(NodeType.STORAGE, State.MAINTENANCE); + String expected = "Maintenance => Up"; + assertEquals(expected, ns.getTextualDifference(new NodeState(NodeType.STORAGE, State.UP)).substring(0, expected.length())); + + NodeState ns1 = new NodeState(NodeType.STORAGE, State.MAINTENANCE).setDescription("Foo bar").setCapacity(1.2).setReliability(2).setDiskCount(4) + .setMinUsedBits(12).setStartTimestamp(5).setDiskState(1, new DiskState(State.DOWN, "bad disk", 1)) + .setDiskState(3, new DiskState(State.UP, "", 2)); + ns1.toString(); + ns1.toString(true); + expected = "Maintenance => Up, capacity: 1.2 => 1.0, reliability: 2 => 1, minUsedBits: 12 => 16, startTimestamp: 5 => 0, disks: 4 => 0, description: Foo bar => "; + assertEquals(expected, ns1.getTextualDifference(new NodeState(NodeType.STORAGE, State.UP)).substring(0, expected.length())); + } + + public void testValidInClusterState() { + try{ + new NodeState(NodeType.DISTRIBUTOR, State.UNKNOWN).verifyValidInSystemState(NodeType.DISTRIBUTOR); + assertTrue("Should not be valid", false); + } catch (Exception e) {} + try{ + new NodeState(NodeType.DISTRIBUTOR, State.UP).setCapacity(3).verifyValidInSystemState(NodeType.DISTRIBUTOR); + assertTrue("Should not be valid", false); + } catch (Exception e) {} + try{ + new NodeState(NodeType.DISTRIBUTOR, State.UP).setReliability(3).verifyValidInSystemState(NodeType.DISTRIBUTOR); + assertTrue("Should not be valid", false); + } catch (Exception e) {} + try{ + new NodeState(NodeType.DISTRIBUTOR, State.UP).setDiskCount(2).verifyValidInSystemState(NodeType.DISTRIBUTOR); + assertTrue("Should not be valid", false); + } catch (Exception e) {} + } +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/state/NodeTest.java b/vdslib/src/test/java/com/yahoo/vdslib/state/NodeTest.java new file mode 100644 index 00000000000..ce2ff075d78 --- /dev/null +++ b/vdslib/src/test/java/com/yahoo/vdslib/state/NodeTest.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vdslib.state; + +/** + * + */ +public class NodeTest extends junit.framework.TestCase { + + public void testEquals() { + Node n1 = new Node(NodeType.STORAGE, 6); + Node n2 = new Node(NodeType.STORAGE, 7); + Node n3 = new Node(NodeType.DISTRIBUTOR, 6); + Node n4 = new Node(NodeType.DISTRIBUTOR, 7); + Node n5 = new Node(NodeType.STORAGE, 6); + Node n6 = new Node(NodeType.STORAGE, 7); + Node n7 = new Node(NodeType.DISTRIBUTOR, 6); + Node n8 = new Node(NodeType.DISTRIBUTOR, 7); + + assertEquals(n1, n5); + assertEquals(n2, n6); + assertEquals(n3, n7); + assertEquals(n4, n8); + assertEquals(n1, n1); + assertEquals(n2, n2); + assertEquals(n3, n3); + assertEquals(n4, n4); + + assertFalse(n1.equals(n2)); + assertFalse(n1.equals(n3)); + assertFalse(n1.equals(n4)); + + assertFalse(n2.equals(n1)); + assertFalse(n2.equals(n3)); + assertFalse(n2.equals(n4)); + + assertFalse(n3.equals(n1)); + assertFalse(n3.equals(n2)); + assertFalse(n3.equals(n4)); + + assertFalse(n4.equals(n1)); + assertFalse(n4.equals(n2)); + assertFalse(n4.equals(n3)); + + assertFalse(n1.equals("class not instance of Node")); + } + + public void testSerialization() { + Node n = new Node(NodeType.STORAGE, 6); + Node other = new Node(n.toString()); + assertEquals(n, other); + assertEquals(n.hashCode(), other.hashCode()); + + n = new Node(NodeType.DISTRIBUTOR, 5); + other = new Node(n.toString()); + assertEquals(n, other); + assertEquals(n.hashCode(), other.hashCode()); + + try { + new Node("nodewithoutdot"); + assertTrue("Method expected to throw IllegalArgumentException", false); + } catch (IllegalArgumentException e) { + assertEquals("Not a legal node string 'nodewithoutdot'.", e.getMessage()); + } + try { + new Node("fleetcontroller.0"); + assertTrue("Method expected to throw IllegalArgumentException", false); + } catch (IllegalArgumentException e) { + assertEquals("Unknown node type 'fleetcontroller'. Legal values are 'storage' and 'distributor'.", e.getMessage()); + } + try { + new Node("storage.badindex"); + assertTrue("Method expected to throw NumberFormatException", false); + } catch (NumberFormatException e) { + assertEquals("For input string: \"badindex\"", e.getMessage()); + } + } + + public void testMaySetWantedState() { + assertTrue(State.UP.maySetWantedStateForThisNodeState(State.DOWN)); + assertTrue(State.UP.maySetWantedStateForThisNodeState(State.MAINTENANCE)); + assertFalse(State.DOWN.maySetWantedStateForThisNodeState(State.UP)); + assertTrue(State.DOWN.maySetWantedStateForThisNodeState(State.MAINTENANCE)); + assertFalse(State.MAINTENANCE.maySetWantedStateForThisNodeState(State.UP)); + assertFalse(State.MAINTENANCE.maySetWantedStateForThisNodeState(State.DOWN)); + } + +} |