001/*
002 * Copyright (c) 2007-2022 The Cascading Authors. All Rights Reserved.
003 *
004 * Project and contact information: https://cascading.wensel.net/
005 *
006 * This file is part of the Cascading project.
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *     http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020
021package cascading.platform;
022
023import java.io.IOException;
024import java.util.Comparator;
025import java.util.HashMap;
026import java.util.Map;
027
028import cascading.flow.FlowConnector;
029import cascading.flow.FlowProcess;
030import cascading.property.AppProps;
031import cascading.scheme.Scheme;
032import cascading.scheme.util.FieldTypeResolver;
033import cascading.tap.SinkMode;
034import cascading.tap.Tap;
035import cascading.tap.partition.Partition;
036import cascading.tuple.Fields;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 *
042 */
043public abstract class TestPlatform
044  {
045  private static final Logger LOG = LoggerFactory.getLogger( TestPlatform.class );
046
047  public static final String CLUSTER_TESTING_PROPERTY = "test.cluster.enabled";
048  public static final String PLATFORM_PREFIX = "platform.";
049
050  private boolean useCluster = false;
051  private boolean enableCluster = true;
052  protected int numMappers = 0;
053  protected int numReducers = 0;
054  protected int numGatherPartitions = 0;
055
056  /**
057   * Method getGlobalProperties fetches all "platform." prefixed system properties.
058   * <p>
059   * Sub-classes of TestPlatform should use these values as overrides before returning from
060   * {@link #getProperties()}.
061   *
062   * @return a Map of properties
063   */
064  public static Map<Object, Object> getGlobalProperties()
065    {
066    HashMap<Object, Object> properties = new HashMap<Object, Object>();
067
068    for( String propertyName : System.getProperties().stringPropertyNames() )
069      {
070      if( propertyName.startsWith( PLATFORM_PREFIX ) )
071        properties.put( propertyName.substring( PLATFORM_PREFIX.length() ), System.getProperty( propertyName ) );
072      }
073
074    if( !properties.isEmpty() )
075      LOG.info( "platform property overrides: ", properties );
076
077    return properties;
078    }
079
080  protected TestPlatform()
081    {
082    enableCluster = Boolean.parseBoolean( System.getProperty( CLUSTER_TESTING_PROPERTY, Boolean.toString( enableCluster ) ) );
083    }
084
085  public String getName()
086    {
087    return getClass().getSimpleName().replaceAll( "^(.*)Platform$", "$1" ).toLowerCase();
088    }
089
090  /**
091   * Prior versions of all the planners had challenges with Merge->GroupBy and related plans.
092   */
093  @Deprecated
094  public boolean supportsGroupByAfterMerge()
095    {
096    return true;
097    }
098
099  public boolean isMapReduce()
100    {
101    return false;
102    }
103
104  public boolean isDAG()
105    {
106    return false;
107    }
108
109  public int getNumMappers()
110    {
111    return numMappers;
112    }
113
114  public void setNumMappers( int numMappers )
115    {
116    this.numMappers = numMappers;
117    }
118
119  public int getNumReducers()
120    {
121    return numReducers;
122    }
123
124  public void setNumReducers( int numReducers )
125    {
126    this.numReducers = numReducers;
127    }
128
129  public int getNumGatherPartitions()
130    {
131    return numGatherPartitions;
132    }
133
134  public void setNumGatherPartitions( int numGatherPartitions )
135    {
136    this.numGatherPartitions = numGatherPartitions;
137    }
138
139  public void setNumMapTasks( Map<Object, Object> properties, int numMapTasks )
140    {
141    // do nothing
142    }
143
144  public void setNumReduceTasks( Map<Object, Object> properties, int numReduceTasks )
145    {
146    // do nothing
147    }
148
149  public void setNumGatherPartitionTasks( Map<Object, Object> properties, int numReduceTasks )
150    {
151    // do nothing
152    }
153
154  public Integer getNumMapTasks( Map<Object, Object> properties )
155    {
156    return null;
157    }
158
159  public Integer getNumReduceTasks( Map<Object, Object> properties )
160    {
161    return null;
162    }
163
164  public Integer getNumGatherPartitionTasks( Map<Object, Object> properties )
165    {
166    return null;
167    }
168
169  public abstract void setUp() throws IOException;
170
171  public abstract Map<Object, Object> getProperties();
172
173  public abstract void tearDown();
174
175  public void setUseCluster( boolean useCluster )
176    {
177    this.useCluster = useCluster;
178    }
179
180  public boolean isUseCluster()
181    {
182    return enableCluster && useCluster;
183    }
184
185  public abstract void copyFromLocal( String inputFile ) throws IOException;
186
187  public abstract void copyToLocal( String outputFile ) throws IOException;
188
189  public abstract boolean remoteExists( String outputFile ) throws IOException;
190
191  public abstract boolean remoteRemove( String outputFile, boolean recursive ) throws IOException;
192
193  public abstract <C> FlowProcess<C> getFlowProcess();
194
195  public abstract FlowConnector getFlowConnector( Map<Object, Object> properties );
196
197  public FlowConnector getFlowConnector()
198    {
199    return getFlowConnector( getProperties() );
200    }
201
202  public abstract Tap getTap( Scheme scheme, String filename, SinkMode mode );
203
204  public Tap getTextFile( Fields sourceFields, String filename )
205    {
206    return getTextFile( sourceFields, filename, SinkMode.KEEP );
207    }
208
209  public Tap getTextFile( String filename )
210    {
211    return getTextFile( filename, SinkMode.KEEP );
212    }
213
214  public Tap getTextFile( String filename, SinkMode mode )
215    {
216    return getTextFile( null, filename, mode );
217    }
218
219  public Tap getTextFile( Fields sourceFields, String filename, SinkMode mode )
220    {
221    return getTextFile( sourceFields, Fields.ALL, filename, mode );
222    }
223
224  public abstract Tap getTextFile( Fields sourceFields, Fields sinkFields, String filename, SinkMode mode );
225
226  public <C, I, O> Tap<? extends C, ? extends I, ? extends O> getDelimitedFile( Fields fields, String delimiter, String filename )
227    {
228    return getDelimitedFile( fields, false, delimiter, "\"", null, filename, SinkMode.KEEP );
229    }
230
231  public Tap getDelimitedFile( Fields fields, String delimiter, String filename, SinkMode mode )
232    {
233    return getDelimitedFile( fields, false, delimiter, "\"", null, filename, mode );
234    }
235
236  public Tap getTabDelimitedFile( Fields fields, String filename, SinkMode mode )
237    {
238    return getDelimitedFile( fields, false, "\t", "\"", null, filename, mode );
239    }
240
241  public Tap getTabDelimitedFile( Fields fields, boolean hasHeader, String filename, SinkMode mode )
242    {
243    return getDelimitedFile( fields, hasHeader, "\t", "\"", null, filename, mode );
244    }
245
246  public Tap getDelimitedFile( Fields fields, boolean hasHeader, String delimiter, String quote, String filename, SinkMode mode )
247    {
248    return getDelimitedFile( fields, hasHeader, delimiter, quote, null, filename, mode );
249    }
250
251  public Tap getDelimitedFile( Fields fields, String delimiter, String quote, String filename, SinkMode mode )
252    {
253    return getDelimitedFile( fields, false, delimiter, quote, null, filename, mode );
254    }
255
256  public Tap getDelimitedFile( Fields fields, String delimiter, Class[] types, String filename, SinkMode mode )
257    {
258    return getDelimitedFile( fields, false, delimiter, "\"", types, filename, mode );
259    }
260
261  public abstract Tap getDelimitedFile( Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode );
262
263  public abstract Tap getDelimitedFile( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode );
264
265  public abstract Tap getDelimitedFile( String delimiter, String quote, FieldTypeResolver fieldTypeResolver, String filename, SinkMode mode );
266
267  public Tap getJSONFile( Fields fields, String filename )
268    {
269    return getJSONFile( fields, filename, SinkMode.KEEP );
270    }
271
272  public abstract Tap getJSONFile( Fields fields, String filename, SinkMode mode );
273
274  public abstract Tap getPartitionTap( Tap sink, Partition partition, int openThreshold );
275
276  public abstract Scheme getTestConfigDefScheme();
277
278  public abstract Scheme getTestFailScheme();
279
280  public abstract Comparator getLongComparator( boolean reverseSort );
281
282  public abstract Comparator getStringComparator( boolean reverseSort );
283
284  public abstract String getHiddenTemporaryPath();
285
286  protected String getApplicationJar()
287    {
288    // mapred.jar is for backwards compatibility with the compatibility suite
289    String property = System.getProperty( "mapred.jar", System.getProperty( AppProps.APP_JAR_PATH ) );
290
291    if( property == null || property.isEmpty() )
292      return null;
293
294    return property;
295    }
296  }