Updates to loading ideas

joseph_ideas_loading.py

 import tensorflow as tf
+import numpy as np
 
-def load_data(filepaths, preproc, batch_size, column = -1, skip_header = false):
+def load_data(filepaths, preproc, batch_size, column = -1, skip_header = False):
     # Loading the dataset file
     x_data = tf.data.TextLineDataset(filepaths, num_parallel_reads=5)
     
@@ -16,9 +17,10 @@ def load_data(filepaths, preproc, batch_size, column = -1, skip_header = false):
 
 def parse_column(x, index):
     columns = tf.strings.split([x], sep=' ')
+    columns = tf.strings.to_number(columns, out_type=tf.float32)
     return columns[index]
 
-def set_shape(x):
+def set_shape(x, Nbeads, dimensions):
     x.set_shape((Nbeads,dimensions))
     return x
 
@@ -26,9 +28,12 @@ def pad_sequences(x):
     x = tf.keras.utils.pad_sequences(x, maxlen=1000, value=-100)
     return x
 
-def process_batch(batch, labels):
+def process_batch(batch, labels, preproc, Nbeads, dimensions):
 
-    batch = batch.map(set_shape)
+    ## either this
+    batch = batch.map(lambda x: set_shape(x, Nbeads, dimensions))
+    ## or this
+    batch = batch.batch(100, drop_remainder = True)
 
     # Apply preprocessing steps
     batch = batch.map(preproc)
@@ -59,7 +64,7 @@ def split_train_test_validation(dataset, train_size, test_size, val_size):
 
 ### MAIN STUFF ###
 
-def main():
+def main(model, cbs, Nbeads, datatype, Knotind, preproc, epochs):
 
     batch_size = 256
 
@@ -82,12 +87,16 @@ def main():
             try:
                 batch = sess.run(next_element)
                 # Perform operations on batch
-                process_batch(batch, labels)
-                train, test, val = split_train_test_validation(batch, 0, 1, 0) # input desired sizes
+                batch_with_labels = process_batch(batch, labels)
+                train, test, val = split_train_test_validation(batch_with_labels, 0, 1, 0) # input desired sizes
+                model.fit(train, epochs, batch_size, callbacks=cbs, validation_data= val)
+                model.evaluate(test)
+
             except tf.errors.OutOfRangeError:
                 # End of dataset
                 break
 
+    
 
 
 ##CHATGPT CODE