Skip to content

GitLab

Commit a8ba4dde authored by Jonathan Juhl's avatar Jonathan Juhl
Browse files

try again

parent 922672f8
Expand all Hide whitespace changes
Inline Side-by-side
execute_sortem.py
View file @ a8ba4dde
......@@ -7,14 +7,14 @@ from os import listdir,rename
from fac_sortem import DynAE
from os import listdir,mkdir
import glob
from os.path import isabs
class control_flow:
def __init__(self,num_gpus,gpu_list,num_cpus,num_clusters,star,deep_NN_batch_size,workdir,half_precision,max_particles,epochs,transfer_learning,verbose):
self.workdir = workdir
self.verbose = verbose
if not isdir(workdir): # check if all dirs are their
......@@ -41,16 +41,15 @@ class control_flow:
def get_star_file_parameters(star_files,path):
counter = []
count = 0
f = 0
names = []
star_files = np.unique(star_files)
labels_list = []
count = 0
for z in star_files:
c = z[::-1].split('/',1)[1][::-1]
#c = z[::-1].split('/',1)[1][::-1]
c = '/emcc/au587640/cryosparc2_data/P139/'
with open(z, newline='') as csvfile:
......@@ -84,52 +83,33 @@ class control_flow:
if self.verbose:
try:
class_num = header.index('_rlnClassNumber')
except:
self.verbose = False
print("the --log true cannot be run as _rlnClassNumber i missing ")
for row in reader:
#current_name = '/emcc/misser11/New_Project/Assets/ParticleStacks/particle_stack_0.mrc'
if len(header)== len(row.split()):
# try:
#current_name = '/emcc/misser11/New_Project/Assets/ParticleStacks/particle_stack_1.mrc'
f+=1
current_name = row.split()[name].split('@')[1]
current_id = row.split()[name].split('@')[0]
if len(names) > 0 and names[0] != join(c,current_name):
counter.append(count)
names.append(join(c,current_name))
count = 0
elif len(names) == 0:
names.append(join(c,current_name))
count +=1
else:
count +=1
if self.verbose:
labels_list.append(int(row.split()[class_num]))
#except: pass
counter.append(count)
tmp_length = np.asarray(counter).sum()
labels_list.append(int(row.split()[class_num]))
current_id = row.split()[name].split('@')[0]
names.append(join(c,current_name))
np.save(join(join(self.workdir, 'particle_stack_dir'),'labels'),np.asarray(labels_list))
return tmp_length,names
return f,np.unique(names)
depth,mrc_paths = get_star_file_parameters(star_files,star)
length,bytes_pr_record = self.get_parameters(mrc_paths)
DynAE( mrc_paths,
......@@ -204,7 +184,7 @@ class control_flow:
return NX[0],NY[0],NZ[0],recordsize
def get_parameters(self,paths):
width,length,depth,record_size_new = self.read(paths[0])
return length,record_size_new
......
fac_sortem.py
View file @ a8ba4dde
This diff is collapsed.
mic_extractor_sortem.py 0 → 100644
View file @ a8ba4dde
import tensorflow as tf
import numpy as np
import mrcfile
output_path = '/emcc/misser11/out_optics_dub_stack.star'
output_dir =
input_star = '/emcc/misser11/out_optics_dub.star'
def open_star_file():
def make_star_file():
def load_big_micrograph(image_path):
def extract_particles(image,x_min,y_min,box_size):
zeros = tf.zeros(tf.shape(x_min)[0])
image_stack = tf.image.crop_and_resize(image,[y_min,x_min,y_min+box_size,x_min+box_size],zeros,[box_size,box_size])
with mrcfile.new('pa', compression='bzip2') as mrc:
for i in range(mrc_paths)
star_file = 'out_optics_dub.star'
def get_star_file_parameters(star_files):
with open(star_files, newline='') as csvfile:
reader = list(csvfile)
header = list(filter(lambda x: ('_rln' == x[0:4] or 'loop_' == x.strip()) and '_rlnMicrographName' not in x,reader))
header = [i.split()[0] for i in header]
header = ['_rlnImageName'] + header[1:]
index = 1
with open('out_star.star', 'w') as cv:
out = []
for i in reader:
if len(header) == len(i.split()):
i = i.split(' ')
new_i = [str("%06d@emcc/misser11/New_Project/Assets/ParticleStacks/particle_stack_1.mrc") % (index,)] +i[1:]
out.append(new_i)
index +=1
cv.write('data_images\n')
cv.write('\n')
cv.write('loop_\n')
for i in header:
cv.write(i+'\n')
cv.write('\n')
for i in out:
cv.write(' '.join(i))
get_star_file_parameters(star_f
\ No newline at end of file
mrc_loader_sortem.py
View file @ a8ba4dde
......@@ -10,7 +10,7 @@ class mrc_loader:
def __init__(self,mrc_path,path,bytes_pr_record,mode,prefix,width,precision,num_cpus,verbose,large_rescale=256,small_rescale=128,batch_size=300,abs_mins=0.08,mins=0.2,maxs=1.0,boolean=None,coordinates=None):
def __init__(self,mrc_path,path,bytes_pr_record,mode,prefix,width,precision,num_cpus,verbose,large_rescale=256,small_rescale=128,batch_size=300,abs_mins=0.08,mins=0.8,maxs=1.0,booleans=None,coordinates=None):
self.num_cpus = num_cpus
self.mrc_paths = mrc_path
......@@ -27,7 +27,7 @@ class mrc_loader:
self.large_rescale= large_rescale
self.bytes_pr_record= bytes_pr_record
self.verbose = verbose
self.boolean = boolean
self.boolean = booleans
self.coordinates = coordinates
if mode == 'w':
......@@ -70,42 +70,44 @@ class mrc_loader:
image = tf.image.per_image_standardization(image)
return image
def small_map(a,x):
if self.boolean is not None:
if len(x) == 2:
x = x[0]
if self.verbose and mode =='predict':
a,y = a
a = tf.image.crop_and_resize(tf.expand_dims(a,axis=0),x,np.asarray([0,0,0,0]),[self.small_rescale,self.small_rescale])
return a,tf.stack([y,y,y,y],axis=0)
a = tf.image.crop_and_resize(tf.expand_dims(a,axis=0),tf.expand_dims(x,axis=0),np.asarray([0]),[self.large_rescale,self.large_rescale])
a = tf.squeeze(a,axis=0)
return a,y
else:
a = tf.image.crop_and_resize(tf.expand_dims(a,axis=0),x,np.asarray([0,0,0,0]),[self.small_rescale,self.small_rescale])
a = tf.image.crop_and_resize(tf.expand_dims(a,axis=0),tf.expand_dims(x,axis=0),np.asarray([0]),[self.large_rescale,self.large_rescale])
a = tf.squeeze(a,axis=0)
return a
def map_image(ins,labels=None):
def filter(x):
return x
def sub_map(images,minval,maxval,rescale):
alphas = tf.random.uniform([1])
s = tf.shape(images)[1]
images =tf.cast( tf.squeeze(images), self.precision )
images = tf.squeeze(tfa.image.rotate(tf.expand_dims(images,axis=2),alphas*2*3.1415))
x = tf.cast(tf.random.uniform([1],minval = minval, maxval = maxval,dtype=tf.float32)*self.image_size,tf.int32)
y = tf.cast(tf.random.uniform([1],minval = minval, maxval = maxval,dtype=tf.float32)*self.image_size,tf.int32)
x = tf.cast(tf.random.uniform([1],minval = minval, maxval = maxval,dtype=tf.float32)*tf.cast(s,self.precision),tf.int32)
y = tf.cast(tf.random.uniform([1],minval = minval, maxval = maxval,dtype=tf.float32)*tf.cast(s,self.precision),tf.int32)
croped_image = tf.image.random_crop(images,tf.squeeze(tf.stack([x,y],axis=0)))
croped_image = tf.expand_dims(croped_image,axis=2)
resized_image = tf.image.resize(croped_image, [rescale,rescale])
resized_image = tf.image.resize(croped_image, [rescale,rescale])
resized_image = tf.image.random_brightness(resized_image,0.1,0.6)
resized_image = tf.image.random_contrast(resized_image,0.1,0.6)
final_image = tf.image.flip_left_right(resized_image)
final_image = final_image/tf.norm(final_image)
croped_image = tf.image.flip_left_right(resized_image)
final_image = croped_image/tf.norm(croped_image)
return final_image
......@@ -116,7 +118,7 @@ class mrc_loader:
if mode == 'contrastive':
print(ins.shape,images.shape)
return ins,images
else:
......@@ -128,54 +130,60 @@ class mrc_loader:
return ins
def filters(a,x):
return x
if mode == 'predict':
data = tf.data.FixedLengthRecordDataset(self.mrc_paths,self.bytes_pr_record,num_parallel_reads=1, header_bytes=1024).repeat().map(premake_image,self.num_cpus)
data = tf.data.FixedLengthRecordDataset(self.mrc_paths,self.bytes_pr_record,num_parallel_reads=1, header_bytes=1024).map(premake_image,self.num_cpus)
else:
data = tf.data.FixedLengthRecordDataset(self.mrc_paths,self.bytes_pr_record,num_parallel_reads=1, header_bytes=1024).repeat().map(premake_image,self.num_cpus).repeat()
if self.verbose:
if mode =='predict':
data = tf.data.FixedLengthRecordDataset(self.mrc_paths,self.bytes_pr_record,num_parallel_reads=1, header_bytes=1024).repeat().map(premake_image,self.num_cpus)#.shuffle(2000).repeat()
lab_1 = tf.data.Dataset.from_tensor_slices(np.load(join(self.path,'labels.npy')))
f = tf.data.Dataset.zip((data,lab_1)).map(lambda a,x: map_image(a,x),self.num_cpus).prefetch(self.batch_size).batch(self.batch_size)
else:
f = data.map(map_image,self.num_cpus).prefetch(self.batch_size).batch(self.batch_size)
return f
if self.boolean is not None and self.coordinates is not None:
coords = tf.data.Dataset.from_tensor_slices(self.coordinates)
coords = tf.data.Dataset.from_tensor_slices(self.coordinates).repeat()
bools = tf.data.Dataset.from_tensor_slices(self.boolean).repeat()
coords = tf.data.Dataset.zip((coords,bools)).filter(lambda a,x: filters(a,x))
zipdata = tf.data.Dataset.zip((data,coords)).map(lambda a,x: small_map(a,x),self.num_cpus)
zipdata = tf.data.Dataset.zip((data,coords)).map(lambda a,x: small_map(a,x),self.num_cpus).unbatch()
data = tf.data.Dataset.zip((zipdata,bools))
bools = tf.data.Dataset.from_tensor_slices(self.boolean)
f = tf.data.Dataset.zip((zipdata,bools)).filter(lambda a,x: x)
return f.map(map_image,self.num_cpus).prefetch(self.batch_size).batch(self.batch_size)
elif self.boolean is None and self.coordinates is not None:
coords = tf.data.Dataset.from_tensor_slices(self.coordinates)
zipdata = tf.data.Dataset.zip((data,coords)).map(lambda a,x: small_map(a,x),self.num_cpus )
coords = tf.data.Dataset.from_tensor_slices(self.coordinates).repeat()
data = tf.data.Dataset.zip((data,coords)).map(lambda a,x: small_map(a,x),self.num_cpus )
if mode =='predict' and self.verbose:
lab_1 = tf.data.Dataset.from_tensor_slices(np.load(join(self.path,'labels.npy')))
f = tf.data.Dataset.zip((data,lab_1)).map(lambda a,x: map_image(a,x),self.num_cpus).prefetch(self.batch_size).batch(self.batch_size)
else:
f = data.map(map_image,self.num_cpus).prefetch(self.batch_size).batch(self.batch_size)
return f
return data.map(map_image,self.num_cpus).prefetch(self.batch_size).batch(self.batch_size)
#return data.map(map_image,self.num_cpus).prefetch(self.batch_size).batch(self.batch_size)
else:
# else:
return data.map(map_image,self.num_cpus).prefetch(self.batch_size).batch(self.batch_size)
#return f.map(map_image,self.num_cpus).prefetch(self.batch_size).batch(self.batch_size)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment