Machine Learning for ISIC Skin Cancer Classification Challenge

def reduce_dimensionality(dataset):
    """ Reduces the dimensionality of dataset (assuming it stored using numpy) """
    data_with_class = np.load(dataset)
    data_no_class, y = extract_features_and_class(data_with_class)

    # Will Seg-fault with regular PCA due to dataset size
    # Somewhat arbitrary batch size here. 
    pca = IncrementalPCA(n_components=N_COMPS)
    num_batches = int(math.ceil(y.shape[0] / float(BATCH_SIZE)))

    for i in xrange(num_batches):
        batch = get_next_batch(data_no_class, i, BATCH_SIZE) 
        pca.partial_fit(batch)

    reduced_data = None
    for i in xrange(num_batches):
        batch = get_next_batch(data_no_class, i, BATCH_SIZE) 
        transformed_chunk = pca.transform(batch)
        if reduced_data == None:
            reduced_data = transformed_chunk
        else:
            reduced_data = np.vstack((reduced_data, transformed_chunk))
    reduced_data_with_class = np.hstack((reduced_data,y))
    return reduced_data_with_class
 
def get_next_batch(data, i, batch_size):
    """ Returns the ith batch of size batch_size from data. 
      If  (i + 1) * batch_size goes past the size of the data, 
      this just returns the remaining rows"""
    if (i + 1) * batch_size > data.shape[0]:
        return data[i * batch_size:data.shape[0], :]
    return data[i * batch_size:(i + 1) * batch_size, :]

def extract_features_and_class(data_with_class):
    """ Seperates the features from the label. 
      Assumes the labels the final column of data_with_class. """
    y = data_with_class[:,-1]
    # Reshape into column vector instead of row
    y_col = y.reshape(y.size,1)
    n_columns = data_with_class.shape[1] - 1
    data_no_class = data_with_class[:,0:n_columns]
    return data_no_class, y_col