问:
Distributed Tensorflow: good example for synchronous training on CPUs

I am new to distributed tensorflow and am looking for a good example to perform synchronous training on CPUs.

I have already tried the Distributed Tensorflow Example and it can perform the asynchronous training successfully over 1 parameter server (1 machine with 1 CPU) and 3 workers (each worker = 1 machine with 1 CPU). However, when it comes to the synchronous training, I am not able to run it correctly, although I have followed the tutorial of
SyncReplicasOptimizer(V1.0 and V2.0).

I have inserted the official SyncReplicasOptimizer code into the working asynchronous training example but the training process is still asynchronous. My detailed code is as follows. Any code relates to synchronous training is within the block of ******.

import tensorflow as tf
import sys
import time

# cluster specification ----------------------------------------------------------------------
parameter_servers = ["xx1.edu:2222"]
workers = ["xx2.edu:2222", "xx3.edu:2222", "xx4.edu:2222"]
cluster = tf.train.ClusterSpec({"ps":parameter_servers, "worker":workers})

# input flags
tf.app.flags.DEFINE_string("job_name", "", "Either 'ps' or 'worker'")
tf.app.flags.DEFINE_integer("task_index", 0, "Index of task within the job")
FLAGS = tf.app.flags.FLAGS

# start a server for a specific task
server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index)

# Parameters  ----------------------------------------------------------------------
N = 3 # number of replicas
learning_rate = 0.001
training_epochs = int(21/N)
batch_size = 100

# Network Parameters
n_input = 784 # MNIST data input (img shape: 28*28)
n_hidden_1 = 256 # 1st layer number of features
n_hidden_2 = 256 # 2nd layer number of features
n_classes = 10 # MNIST total classes (0-9 digits)

if FLAGS.job_name == "ps":
    server.join()
    print("--- Parameter Server Ready ---")
elif FLAGS.job_name == "worker":
    # Import MNIST data
    from tensorflow.examples.tutorials.mnist import input_data
    mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
    # Between-graph replication
    with tf.device(tf.train.replica_device_setter(
        worker_device="/job:worker/task:%d" % FLAGS.task_index,
        cluster=cluster)):
        # count the number of updates
        global_step = tf.get_variable('global_step', [],
                                      initializer = tf.constant_initializer(0),
                                      trainable = False,
                                      dtype = tf.int32)
        # tf Graph input
        x = tf.placeholder("float", [None, n_input])
        y = tf.placeholder("float", [None, n_classes])

        # Create model
        def multilayer_perceptron(x, weights, biases):
            # Hidden layer with RELU activation
            layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
            layer_1 = tf.nn.relu(layer_1)
            # Hidden layer with RELU activation
            layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
            layer_2 = tf.nn.relu(layer_2)
            # Output layer with linear activation
            out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
            return out_layer

        # Store layers weight & bias
        weights = {
            'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
            'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
            'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
        }
        biases = {
            'b1': tf.Variable(tf.random_normal([n_hidden_1])),
            'b2': tf.Variable(tf.random_normal([n_hidden_2])),
            'out': tf.Variable(tf.random_normal([n_classes]))
        }

        # Construct model
        pred = multilayer_perceptron(x, weights, biases)

        # Define loss and optimizer
        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))

        # ************************* SyncReplicasOpt Version 1.0 *****************************************************
        ''' This optimizer collects gradients from all replicas, "summing" them,
        then applying them to the variables in one shot, after which replicas can fetch the new variables and continue. '''
        # Create any optimizer to update the variables, say a simple SGD
        opt = tf.train.AdamOptimizer(learning_rate=learning_rate)

        # Wrap the optimizer with sync_replicas_optimizer with N replicas: at each step the optimizer collects N gradients before applying to variables.
        opt = tf.train.SyncReplicasOptimizer(opt, replicas_to_aggregate=N,
                                        replica_id=FLAGS.task_index, total_num_replicas=N)

        # Now you can call `minimize()` or `compute_gradients()` and `apply_gradients()` normally
        train = opt.minimize(cost, global_step=global_step)

        # You can now call get_init_tokens_op() and get_chief_queue_runner().
        # Note that get_init_tokens_op() must be called before creating session
        # because it modifies the graph.
        init_token_op = opt.get_init_tokens_op()
        chief_queue_runner = opt.get_chief_queue_runner()
        # **************************************************************************************

        # Test model
        correct = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
        accuracy = tf.reduce_mean(tf.cast(correct, "float"))

        # Initializing the variables
        init_op = tf.initialize_all_variables()
        print("---Variables initialized---")

    # **************************************************************************************
    is_chief = (FLAGS.task_index == 0)
    # Create a "supervisor", which oversees the training process.
    sv = tf.train.Supervisor(is_chief=is_chief,
                             logdir="/tmp/train_logs",
                             init_op=init_op,
                             global_step=global_step,
                             save_model_secs=600)
    # **************************************************************************************

    with sv.prepare_or_wait_for_session(server.target) as sess:
        # **************************************************************************************
        # After the session is created by the Supervisor and before the main while loop:
        if is_chief:
            sv.start_queue_runners(sess, [chief_queue_runner])
            # Insert initial tokens to the queue.
            sess.run(init_token_op)
        # **************************************************************************************
        # Statistics
        net_train_t = 0
        # Training
        for epoch in range(training_epochs):
            total_batch = int(mnist.train.num_examples/batch_size)
            # Loop over all batches
            for i in range(total_batch):
                batch_x, batch_y = mnist.train.next_batch(batch_size)
                # ======== net training time ========
                begin_t = time.time()
                sess.run(train, feed_dict={x: batch_x, y: batch_y})
                end_t = time.time()
                net_train_t += (end_t - begin_t)
                # ===================================
            # Calculate training accuracy
            # acc = sess.run(accuracy, feed_dict={x: mnist.train.images, y: mnist.train.labels})
            # print("Epoch:", '%04d' % (epoch+1), " Train Accuracy =", acc)
            print("Epoch:", '%04d' % (epoch+1))
        print("Training Finished!")
        print("Net Training Time: ", net_train_t, "second")
        # Testing
        print("Testing Accuracy = ", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))

    sv.stop()
    print("done")
```

Anything wrong with my code? Or can I have a good example to follow?

答:
One issue is that you need to specify an aggregation_method in the minimize method for it to run synchronously,</p>

```<code>train = opt.minimize(cost, global_step=global_step, aggregation_method=tf.AggregationMethod.ADD_N)
</code>```

Body:
I am new to distributed tensorflow and am looking for a good example to perform synchronous training on CPUs.

I have already tried the Distributed Tensorflow Example and it can perform the asynchronous training successfully over 1 parameter server (1 machine with 1 CPU) and 3 workers (each worker = 1 machine with 1 CPU). However, when it comes to the synchronous training, I am not able to run it correctly, although I have followed the tutorial of
SyncReplicasOptimizer(V1.0 and V2.0).

I have inserted the official SyncReplicasOptimizer code into the working asynchronous training example but the training process is still asynchronous. My detailed code is as follows. Any code relates to synchronous training is within the block of ******.

import tensorflow as tf
import sys
import time

# cluster specification ----------------------------------------------------------------------
parameter_servers = ["xx1.edu:2222"]
workers = ["xx2.edu:2222", "xx3.edu:2222", "xx4.edu:2222"]
cluster = tf.train.ClusterSpec({"ps":parameter_servers, "worker":workers})

# input flags
tf.app.flags.DEFINE_string("job_name", "", "Either 'ps' or 'worker'")
tf.app.flags.DEFINE_integer("task_index", 0, "Index of task within the job")
FLAGS = tf.app.flags.FLAGS

# start a server for a specific task
server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index)

# Parameters  ----------------------------------------------------------------------
N = 3 # number of replicas
learning_rate = 0.001
training_epochs = int(21/N)
batch_size = 100

# Network Parameters
n_input = 784 # MNIST data input (img shape: 28*28)
n_hidden_1 = 256 # 1st layer number of features
n_hidden_2 = 256 # 2nd layer number of features
n_classes = 10 # MNIST total classes (0-9 digits)

if FLAGS.job_name == "ps":
    server.join()
    print("--- Parameter Server Ready ---")
elif FLAGS.job_name == "worker":
    # Import MNIST data
    from tensorflow.examples.tutorials.mnist import input_data
    mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
    # Between-graph replication
    with tf.device(tf.train.replica_device_setter(
        worker_device="/job:worker/task:%d" % FLAGS.task_index,
        cluster=cluster)):
        # count the number of updates
        global_step = tf.get_variable('global_step', [],
                                      initializer = tf.constant_initializer(0),
                                      trainable = False,
                                      dtype = tf.int32)
        # tf Graph input
        x = tf.placeholder("float", [None, n_input])
        y = tf.placeholder("float", [None, n_classes])

        # Create model
        def multilayer_perceptron(x, weights, biases):
            # Hidden layer with RELU activation
            layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
            layer_1 = tf.nn.relu(layer_1)
            # Hidden layer with RELU activation
            layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
            layer_2 = tf.nn.relu(layer_2)
            # Output layer with linear activation
            out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
            return out_layer

        # Store layers weight & bias
        weights = {
            'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
            'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
            'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
        }
        biases = {
            'b1': tf.Variable(tf.random_normal([n_hidden_1])),
            'b2': tf.Variable(tf.random_normal([n_hidden_2])),
            'out': tf.Variable(tf.random_normal([n_classes]))
        }

        # Construct model
        pred = multilayer_perceptron(x, weights, biases)

        # Define loss and optimizer
        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))

        # ************************* SyncReplicasOpt Version 1.0 *****************************************************
        ''' This optimizer collects gradients from all replicas, "summing" them,
        then applying them to the variables in one shot, after which replicas can fetch the new variables and continue. '''
        # Create any optimizer to update the variables, say a simple SGD
        opt = tf.train.AdamOptimizer(learning_rate=learning_rate)

        # Wrap the optimizer with sync_replicas_optimizer with N replicas: at each step the optimizer collects N gradients before applying to variables.
        opt = tf.train.SyncReplicasOptimizer(opt, replicas_to_aggregate=N,
                                        replica_id=FLAGS.task_index, total_num_replicas=N)

        # Now you can call `minimize()` or `compute_gradients()` and `apply_gradients()` normally
        train = opt.minimize(cost, global_step=global_step)

        # You can now call get_init_tokens_op() and get_chief_queue_runner().
        # Note that get_init_tokens_op() must be called before creating session
        # because it modifies the graph.
        init_token_op = opt.get_init_tokens_op()
        chief_queue_runner = opt.get_chief_queue_runner()
        # **************************************************************************************

        # Test model
        correct = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
        accuracy = tf.reduce_mean(tf.cast(correct, "float"))

        # Initializing the variables
        init_op = tf.initialize_all_variables()
        print("---Variables initialized---")

    # **************************************************************************************
    is_chief = (FLAGS.task_index == 0)
    # Create a "supervisor", which oversees the training process.
    sv = tf.train.Supervisor(is_chief=is_chief,
                             logdir="/tmp/train_logs",
                             init_op=init_op,
                             global_step=global_step,
                             save_model_secs=600)
    # **************************************************************************************

    with sv.prepare_or_wait_for_session(server.target) as sess:
        # **************************************************************************************
        # After the session is created by the Supervisor and before the main while loop:
        if is_chief:
            sv.start_queue_runners(sess, [chief_queue_runner])
            # Insert initial tokens to the queue.
            sess.run(init_token_op)
        # **************************************************************************************
        # Statistics
        net_train_t = 0
        # Training
        for epoch in range(training_epochs):
            total_batch = int(mnist.train.num_examples/batch_size)
            # Loop over all batches
            for i in range(total_batch):
                batch_x, batch_y = mnist.train.next_batch(batch_size)
                # ======== net training time ========
                begin_t = time.time()
                sess.run(train, feed_dict={x: batch_x, y: batch_y})
                end_t = time.time()
                net_train_t += (end_t - begin_t)
                # ===================================
            # Calculate training accuracy
            # acc = sess.run(accuracy, feed_dict={x: mnist.train.images, y: mnist.train.labels})
            # print("Epoch:", '%04d' % (epoch+1), " Train Accuracy =", acc)
            print("Epoch:", '%04d' % (epoch+1))
        print("Training Finished!")
        print("Net Training Time: ", net_train_t, "second")
        # Testing
        print("Testing Accuracy = ", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))

    sv.stop()
    print("done")

Anything wrong with my code? Or can I have a good example to follow?

Body:
One issue is that you need to specify an aggregation_method in the minimize method for it to run synchronously,

train = opt.minimize(cost, global_step=global_step, aggregation_method=tf.AggregationMethod.ADD_N)


来源: stackexchange

问:
Opening explorer from bash/linux subsystem in Windows 10
Is there a command to open the current directory in windows file explorer from bash (or zsh) using bash on ubuntu on windows? I think I saw somewhere you can do explorer . but it tells me No command 'explorer' found

答:
This works well for me on my GIT bash on windows 10.</p>

```<code>$ which explorer
/c/Windows/explorer
$ explorer .
</code>```

ContentLicense: CC BY-SA 3.0

Body:
This works well for me on my GIT bash on windows 10.

$ which explorer
/c/Windows/explorer
$ explorer .


问:
Swift: Simplify menu button logic
I have a game in Swift where each of my levels is a separate class inherited from the base GameScene class (it's a lot easier this way for what I'm doing, don't judge me).
I also have a menu which has a button for each level. This is how the buttons loads the level:

```<code>override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
  if let t = touches.first {
    let node = atPoint(t.location(in: self))
    if let name = node.name {
      let newScene: GameScene!
      switch Int(name)! {
      case 1:
        newScene = Level1(size: frame.size)
      case 2:
        newScene = Level2(size: frame.size)
      case 3:
        newScene = Level3(size: frame.size)
      case 4:
        newScene = Level4(size: frame.size)
      case 5:
        newScene = Level5(size: frame.size)
      case 6:
        newScene = Level6(size: frame.size)
      case 7:
        newScene = Level7(size: frame.size)
      default:
        newScene = Level1(size: frame.size)
      }
      view?.presentScene(newScene, transition: .crossFade(withDuration: 0.5))
    }
  }
}
</code>```

To me, this switch looks incredibly ugly and pointless, but I can't think of a way to avoid it. I was hoping someone here could help me out with this, I just can't think of a better alternative.

答:
You could construct your Level's class name as a string "Level\(name)" first and then get the actual class you need by passing it's name to the function:

```<code>func classFromString(_ className: String) -> AnyClass! {
    let namespace = Bundle.main.infoDictionary!["CFBundleExecutable"] as! String
    let cls: AnyClass = NSClassFromString("\(namespace).\(className)")!
    return cls
}
</code>```

Usage example:

```<code>let className = "Level1"
let levelInstance = (classFromString(className) as! GameScene).init(size: frame.size)
</code>```

... but of course  the best advice would be to avoid such kind of architecture

Title: Swift: Simplify menu button logic

Body:
I have a game in Swift where each of my levels is a separate class inherited from the base GameScene class (it's a lot easier this way for what I'm doing, don't judge me).
I also have a menu which has a button for each level. This is how the buttons loads the level:

override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
  if let t = touches.first {
    let node = atPoint(t.location(in: self))
    if let name = node.name {
      let newScene: GameScene!
      switch Int(name)! {
      case 1:
        newScene = Level1(size: frame.size)
      case 2:
        newScene = Level2(size: frame.size)
      case 3:
        newScene = Level3(size: frame.size)
      case 4:
        newScene = Level4(size: frame.size)
      case 5:
        newScene = Level5(size: frame.size)
      case 6:
        newScene = Level6(size: frame.size)
      case 7:
        newScene = Level7(size: frame.size)
      default:
        newScene = Level1(size: frame.size)
      }
      view?.presentScene(newScene, transition: .crossFade(withDuration: 0.5))
    }
  }
}

To me, this switch looks incredibly ugly and pointless, but I can't think of a way to avoid it. I was hoping someone here could help me out with this, I just can't think of a better alternative.

Body:
You could construct your Level's class name as a string "Level\(name)" first and then get the actual class you need by passing it's name to the function:

func classFromString(_ className: String) -> AnyClass! {
    let namespace = Bundle.main.infoDictionary!["CFBundleExecutable"] as! String
    let cls: AnyClass = NSClassFromString("\(namespace).\(className)")!
    return cls
}

Usage example:

let className = "Level1"
let levelInstance = (classFromString(className) as! GameScene).init(size: frame.size)

... but of course  the best advice would be to avoid such kind of architecture


问:
Generate closest teams based on employee schedules C#
I am given a csv of employee schedules with columns:

```<code>employee ID, first last name, sunday schedule, monday schedule, ... , saturday schedule
</code>```

1 week schedule for each employee. I've attached a screenshot of a portion of the csv file. The total file has around 300 rows.

I need to generate teams of 15 based on the employees' schedules (locations don't matter) so that the employees on each team have the closest schedules to each other. Pseudocode of what I have tried:

parse csv file into array of schedules (my own struct definition)

match employees who have the same exact schedule into teams (creates ~5 full sized teams, 20 - 25 half filled teams, leaves ~50 schedules who don't match with anyone)

for i = 1 to 14, for each member of teams of size i, find the team with the closest schedule (as a whole) and add the member to that team. Once a team reaches size 15, mark them as "done".

This worked somewhat but definitely did not give me the best teams. My question is does anyone know a better way to do this? Pseudocode or just a general idea will help, thanks.

EDIT: Here is an example of the formula of comparison
The comparison is based on half hour blocks of difference between the agents schedules. Agent 25 has a score of 16 because he has a difference of 8 half hours with Agent 23 and 24. The team's total score is 32 based on everyone's scores added together.

Not all agents work 8 hour days, and many have different days off, which have the greatest effect on their "closeness" score. Also, a few agents have a different schedule on a certain day than their normal schedule. For example, one agent might work 7am - 3pm on mondays but work 8am - 4pm on tuesday - friday.

答:
Can't say for sure about the schedules, but in string algorithms you can find an edit distance calculation. The idea is to define number of operations you need to perform to get one string from another. For example, distance between kitten and sitting is 3, 2 for substitutions and 1 for deletion. I think that you can define a metric between two employees' schedule in similar way.

Now, after you have a distance function, you may start a clusterization. The k-means algorithm may be a good start for you, but it's main disadvantage is that the number of groups is fixed initially. But I think that you can easily adjust general logic for your needs. After that, you may try some additional ways to cluster your data, but you really should start with your distance function, and then simply optimize it on your employee records.

Body:
I am given a csv of employee schedules with columns:

employee ID, first last name, sunday schedule, monday schedule, ... , saturday schedule

1 week schedule for each employee. I've attached a screenshot of a portion of the csv file. The total file has around 300 rows.

I need to generate teams of 15 based on the employees' schedules (locations don't matter) so that the employees on each team have the closest schedules to each other. Pseudocode of what I have tried:

parse csv file into array of schedules (my own struct definition)

match employees who have the same exact schedule into teams (creates ~5 full sized teams, 20 - 25 half filled teams, leaves ~50 schedules who don't match with anyone)

for i = 1 to 14, for each member of teams of size i, find the team with the closest schedule (as a whole) and add the member to that team. Once a team reaches size 15, mark them as "done".

This worked somewhat but definitely did not give me the best teams. My question is does anyone know a better way to do this? Pseudocode or just a general idea will help, thanks.

EDIT: Here is an example of the formula of comparison
The comparison is based on half hour blocks of difference between the agents schedules. Agent 25 has a score of 16 because he has a difference of 8 half hours with Agent 23 and 24. The team's total score is 32 based on everyone's scores added together.

Not all agents work 8 hour days, and many have different days off, which have the greatest effect on their "closeness" score. Also, a few agents have a different schedule on a certain day than their normal schedule. For example, one agent might work 7am - 3pm on mondays but work 8am - 4pm on tuesday - friday.


问:
Sharing a struct safely amongst multiple threads
Under Windows/C++, if I have a struct:

```<code>struct ListItem {
    ListItem* next;
    ListItem* prev;
    ...
}
</code>```

together with multiple threads running in both the main process and in several dynamically loaded DLLs, and all of those threads need to share the above struct, how do I stop them stomping on each other? Something like:

```<code>ListItem* list = ...

A

list->next = ...

B
</code>```

What do I put at A and B to prevent no more than one thread at a time running list->next = ...?

答: Depending on what you need the structure for, you can use some lockless data structure, like Win API interlocked SList. This way even when multiple threads are performing operations, the data structure is still consistent.</p>

ContentLicense: CC BY-SA 2.5

Body:
Under Windows/C++, if I have a struct:

struct ListItem {
    ListItem* next;
    ListItem* prev;
    ...
}

together with multiple threads running in both the main process and in several dynamically loaded DLLs, and all of those threads need to share the above struct, how do I stop them stomping on each other? Something like:

ListItem* list = ...

A

list->next = ...

B

What do I put at A and B to prevent no more than one thread at a time running list->next = ...?


问:
lib OSMesa off-screen context creation fails in C++, but only when statically linked
I made a C++ tool for off-screen rendering of 3D models. The rendering is done using OSMesa library.

The software was working flawlessly for more than a year, and I stopped to make updates to it something like 6 months ago. In the meanwhile my development environment was updated multiple times.

Now I was compiling it again and found an unexpected bug.

The plain version of the software was still working as expected, but the statically linked one is segfaulting.

I'm assuming that the error is mine in the OSmesa configuration/compilation/linking procedure and not in the library code, but any advice about better debugging of the segmentation fault is appreciated.

Having tried numerous variations of the compilation process without success, I'm now quite stuck.
Anyone can see something stupid I'm doing in some of the steps described below?

I recompiled a static version of the OSmesa library with the same version of the shared library that is working in my system (12.0.6), disabling all the non-needed features (using an Ubuntu based system, no static version of OSmesa lib is available from repositories):

```
./configure \
    --disable-xvmc \
    --disable-glx \
    --disable-dri \
    --with-dri-drivers="" \
    --with-gallium-drivers="" \
    --disable-shared-glapi \
    --disable-egl \
    --with-egl-platforms="" \
    --enable-osmesa \
    --enable-gallium-llvm=no \
    --disable-gles1 \
    --disable-gles2 \
    --enable-static \
    --disable-shared
```

This is the compile command of my off-screen rendering tool:

```
g++ -std=c++11 -Wall -O3 -g -static -static-libgcc -static-libstdc++ ./src/measure_model.cpp model.o thumbnail.o -o measure_model_debug -pthread -lOSMesa -ldl -lm -lpng -lz -lcrypto
```

This is a warning that I was getting by statically compiling using OSMesa, and it was present even a year ago with the working static binary:

```
/home/XXX/XXX/backend/lambda/mesa/mesa-12.0.6/src/mesa/main/dlopen.h:52: warning: Using 'dlopen' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
```

This is what I get from running the tool:

```
Segmentation fault (core dumped)
```

But no segmentation fault is produced if I simply skip the OSmesa context creation step (and obviously all the 3D rendering)

This is the backtrace:

```
#0  0x0000000000000000 in ?? ()
#1  0x00000000004af20a in mtx_init (type=4, mtx=0xe10f70) at ../../include/c11/threads_posix.h:215
#2  _mesa_NewHashTable () at main/hash.c:135
#3  0x000000000052f295 in _mesa_alloc_shared_state (ctx=ctx@entry=0xdcc9b0) at main/shared.c:67
#4  0x000000000046e717 in _mesa_initialize_context (ctx=ctx@entry=0xdcc9b0, api=api@entry=API_OPENGL_COMPAT, visual=, share_list=share_list@entry=0x0, driverFunctions=driverFunctions@entry=0x7fffffffcd40) at main/context.c:1192
#5  0x000000000046c870 in OSMesaCreateContextAttribs (attribList=attribList@entry=0x7fffffffd290, sharelist=) at osmesa.c:834
#6  0x000000000046ccdc in OSMesaCreateContextExt (format=, depthBits=, stencilBits=, accumBits=, sharelist=) at osmesa.c:660
#7  0x0000000000468742 in generate_thumbnail(Model*, Json::Value) ()
#8  0x0000000000401c7d in main (argc=, argv=) at ./src/measure_model.cpp:107
```

A statically linked binary is a strict requirement.

The segmentation fault is happening on the same machine I use to compile the tool (OSmesa static lib is compiled in the same machine too), but no segmentation fault in the non-statically linked version of the same tool.

答:
This is what I get from running the tool:
     Segmentation fault (core dumped)

But no segmentation fault is produced if I simply skip the OSmesa context creation step (and obviously all the 3D rendering)

So, there is some problem from OSmesa creation. With your backtrace we can see that top function was executed from EIP of zero (jump to NULL / call of NULL), so there is call of some function in mtx_init, which is part of OS Mesa context creating.

```<code>#0  0x0000000000000000 in ?? ()
#1  0x00000000004af20a in mtx_init (type=4, mtx=0xe10f70) at ../../include/c11/threads_posix.h:215
#2  _mesa_NewHashTable () at main/hash.c:135
#3  0x000000000052f295 in _mesa_alloc_shared_state (ctx=ctx@entry=0xdcc9b0) at main/shared.c:67
#4  0x000000000046e717 in _mesa_initialize_context (ctx=ctx@entry=0xdcc9b0, api=api@entry=API_OPENGL_COMPAT, visual=, share_list=share_list@entry=0x0, driverFunctions=driverFunctions@entry=0x7fffffffcd40) at main/context.c:1192
#5  0x000000000046c870 in OSMesaCreateContextAttribs (attribList=attribList@entry=0x7fffffffd290, sharelist=) at osmesa.c:834
#6  0x000000000046ccdc in OSMesaCreateContextExt (format=, depthBits=, stencilBits=, accumBits=, sharelist=) at osmesa.c:660
#7  0x0000000000468742 in generate_thumbnail(Model*, Json::Value) ()
#8  0x0000000000401c7d in main (argc=, argv=) at ./src/measure_model.cpp:107
</code>```

What was the function? According to online sources of include/c11/threads_posix.h: mtx_init() on github, there are only calls to pthread_mutex_init, pthread_mutexattr_init and several other mutex related functions of libpthread (-lpthread).

Why there was produced call to NULL instead of real function? Probably due to using static linkage of glibc and/or libpthread. Exact problem is still unidentified at this moment (I was able to found report of statically linked libpthread.a into some shared lib which is incorrect and will never work).

In your case there is only alias (strong one) of pthread_mutex_init in glibc/nptl/pthread_mutex_init.c (line 150) strong_alias (__pthread_mutex_init, pthread_mutex_init) and there may be some weak alias of the symbol in the glibc itself, probably uninitialized. Some was wrong in your linking options or/and in ld mind and he did not find/link the nptl/pthread_mutex_init.o (it is part of libpthread.a archive) with real symbol into final executable (ld often skips unused/unneeded objects of .a archives and don't link them into final executable), keeping the relocation pointing into NULL. Some expert of glibc may know, Employed Russian is one of experts on SO.

I suggest to link statically only to your internal libs or probably also to normal non-system libs like mesa (you may use -Wl,-Bstatic -lyour_lib -Wl,-Bdynamic options to temporary change linkage to static for libs listed between; or use cheat option of -l: as -l:libYour_lib.a found by Radek in the same q.). But do not link statically to most basic libs of glibc like libc, libpthread, librt (there are some problems in static linking of glibc when nss is used: target system must have exact same version of dynamic glibc to enable nss to work).

If you want to pack your application for older machines and you needs some features of glibc you may also try to pack your own version of shared glibc libs with your application; put them to some subdirectory, add rpath option of linker to change library search paths, also change INTERP section from default ABI ld-linux.so.2 loader to your own copy of ld-linux.so.2 from your version of glibc, ... And you still will have problems with too old kernels, as newer glibcs requires some modern features (syscalls, structs) of rather new kernel.

Or you can pack your application into some sort of container like Docker, or some other isolation solution (or chroot?) to always have your versions of libs...

UPDATE: Just found report of similar bt with NULL instead of mutex implementation from nptl: https://bugzilla.redhat.com/show_bug.cgi?id=163083 "Statically linked C++ program using pthreads will segfault" (2005-2007) pthread_mutex_init(&lock, NULL); g++ -g -static foo.cpp -o foo -lpthread where #0  0x00000000 in ?? () #1  0x08048232 in main () at foo.cpp:7

This is apparently due to certain pthreads functions not being included in the output executable.  This bug may duplicate #115157, and I apologize if so, but hopefully the included test case will be useful.

Additional info:

The suggestion in #115157 to forcibly link in all of libpthread.a is a valid workaround.

https://bugzilla.redhat.com/show_bug.cgi?id=115157 "executables linked statically with /usr/lib/nptl/libpthread.a fail" - 2004-2009 CLOSED WONTFIX

Jakub Jelinek 2004-10-29 05:26:10 EDT

First of all, avoid -static if you can, it only creates problems,
  both portability wise and others as well.

If you really need to create statically linked binary with -lpthread
  linked in, then just use -Wl,--whole-archive -lpthread -Wl,--no-whole-archive
  instead of -pthread.  Anything else has really many problems.

Body:
I made a C++ tool for off-screen rendering of 3D models. The rendering is done using OSMesa library.

The software was working flawlessly for more than a year, and I stopped to make updates to it something like 6 months ago. In the meanwhile my development environment was updated multiple times.

Now I was compiling it again and found an unexpected bug.

The plain version of the software was still working as expected, but the statically linked one is segfaulting.

I'm assuming that the error is mine in the OSmesa configuration/compilation/linking procedure and not in the library code, but any advice about better debugging of the segmentation fault is appreciated.

Having tried numerous variations of the compilation process without success, I'm now quite stuck.
Anyone can see something stupid I'm doing in some of the steps described below?

I recompiled a static version of the OSmesa library with the same version of the shared library that is working in my system (12.0.6), disabling all the non-needed features (using an Ubuntu based system, no static version of OSmesa lib is available from repositories):

./configure \
    --disable-xvmc \
    --disable-glx \
    --disable-dri \
    --with-dri-drivers="" \
    --with-gallium-drivers="" \
    --disable-shared-glapi \
    --disable-egl \
    --with-egl-platforms="" \
    --enable-osmesa \
    --enable-gallium-llvm=no \
    --disable-gles1 \
    --disable-gles2 \
    --enable-static \
    --disable-shared

This is the compile command of my off-screen rendering tool:

g++ -std=c++11 -Wall -O3 -g -static -static-libgcc -static-libstdc++ ./src/measure_model.cpp model.o thumbnail.o -o measure_model_debug -pthread -lOSMesa -ldl -lm -lpng -lz -lcrypto

This is a warning that I was getting by statically compiling using OSMesa, and it was present even a year ago with the working static binary:

/home/XXX/XXX/backend/lambda/mesa/mesa-12.0.6/src/mesa/main/dlopen.h:52: warning: Using 'dlopen' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking

This is what I get from running the tool:

Segmentation fault (core dumped)

But no segmentation fault is produced if I simply skip the OSmesa context creation step (and obviously all the 3D rendering)

This is the backtrace:

#0  0x0000000000000000 in ?? ()
#1  0x00000000004af20a in mtx_init (type=4, mtx=0xe10f70) at ../../include/c11/threads_posix.h:215
#2  _mesa_NewHashTable () at main/hash.c:135
#3  0x000000000052f295 in _mesa_alloc_shared_state (ctx=ctx@entry=0xdcc9b0) at main/shared.c:67
#4  0x000000000046e717 in _mesa_initialize_context (ctx=ctx@entry=0xdcc9b0, api=api@entry=API_OPENGL_COMPAT, visual=, share_list=share_list@entry=0x0, driverFunctions=driverFunctions@entry=0x7fffffffcd40) at main/context.c:1192
#5  0x000000000046c870 in OSMesaCreateContextAttribs (attribList=attribList@entry=0x7fffffffd290, sharelist=) at osmesa.c:834
#6  0x000000000046ccdc in OSMesaCreateContextExt (format=, depthBits=, stencilBits=, accumBits=, sharelist=) at osmesa.c:660
#7  0x0000000000468742 in generate_thumbnail(Model*, Json::Value) ()
#8  0x0000000000401c7d in main (argc=, argv=) at ./src/measure_model.cpp:107

A statically linked binary is a strict requirement.

The segmentation fault is happening on the same machine I use to compile the tool (OSmesa static lib is compiled in the same machine too), but no segmentation fault in the non-statically linked version of the same tool.

Body:
This is what I get from running the tool:
     Segmentation fault (core dumped)

But no segmentation fault is produced if I simply skip the OSmesa context creation step (and obviously all the 3D rendering)

So, there is some problem from OSmesa creation. With your backtrace we can see that top function was executed from EIP of zero (jump to NULL / call of NULL), so there is call of some function in mtx_init, which is part of OS Mesa context creating.

#0  0x0000000000000000 in ?? ()
#1  0x00000000004af20a in mtx_init (type=4, mtx=0xe10f70) at ../../include/c11/threads_posix.h:215
#2  _mesa_NewHashTable () at main/hash.c:135
#3  0x000000000052f295 in _mesa_alloc_shared_state (ctx=ctx@entry=0xdcc9b0) at main/shared.c:67
#4  0x000000000046e717 in _mesa_initialize_context (ctx=ctx@entry=0xdcc9b0, api=api@entry=API_OPENGL_COMPAT, visual=, share_list=share_list@entry=0x0, driverFunctions=driverFunctions@entry=0x7fffffffcd40) at main/context.c:1192
#5  0x000000000046c870 in OSMesaCreateContextAttribs (attribList=attribList@entry=0x7fffffffd290, sharelist=) at osmesa.c:834
#6  0x000000000046ccdc in OSMesaCreateContextExt (format=, depthBits=, stencilBits=, accumBits=, sharelist=) at osmesa.c:660
#7  0x0000000000468742 in generate_thumbnail(Model*, Json::Value) ()
#8  0x0000000000401c7d in main (argc=, argv=) at ./src/measure_model.cpp:107

What was the function? According to online sources of include/c11/threads_posix.h: mtx_init() on github, there are only calls to pthread_mutex_init, pthread_mutexattr_init and several other mutex related functions of libpthread (-lpthread).

Why there was produced call to NULL instead of real function? Probably due to using static linkage of glibc and/or libpthread. Exact problem is still unidentified at this moment (I was able to found report of statically linked libpthread.a into some shared lib which is incorrect and will never work).

In your case there is only alias (strong one) of pthread_mutex_init in glibc/nptl/pthread_mutex_init.c (line 150) strong_alias (__pthread_mutex_init, pthread_mutex_init) and there may be some weak alias of the symbol in the glibc itself, probably uninitialized. Some was wrong in your linking options or/and in ld mind and he did not find/link the nptl/pthread_mutex_init.o (it is part of libpthread.a archive) with real symbol into final executable (ld often skips unused/unneeded objects of .a archives and don't link them into final executable), keeping the relocation pointing into NULL. Some expert of glibc may know, Employed Russian is one of experts on SO.

I suggest to link statically only to your internal libs or probably also to normal non-system libs like mesa (you may use -Wl,-Bstatic -lyour_lib -Wl,-Bdynamic options to temporary change linkage to static for libs listed between; or use cheat option of -l: as -l:libYour_lib.a found by Radek in the same q.). But do not link statically to most basic libs of glibc like libc, libpthread, librt (there are some problems in static linking of glibc when nss is used: target system must have exact same version of dynamic glibc to enable nss to work).

If you want to pack your application for older machines and you needs some features of glibc you may also try to pack your own version of shared glibc libs with your application; put them to some subdirectory, add rpath option of linker to change library search paths, also change INTERP section from default ABI ld-linux.so.2 loader to your own copy of ld-linux.so.2 from your version of glibc, ... And you still will have problems with too old kernels, as newer glibcs requires some modern features (syscalls, structs) of rather new kernel.

Or you can pack your application into some sort of container like Docker, or some other isolation solution (or chroot?) to always have your versions of libs...

UPDATE: Just found report of similar bt with NULL instead of mutex implementation from nptl: https://bugzilla.redhat.com/show_bug.cgi?id=163083 "Statically linked C++ program using pthreads will segfault" (2005-2007) pthread_mutex_init(&lock, NULL); g++ -g -static foo.cpp -o foo -lpthread where #0  0x00000000 in ?? () #1  0x08048232 in main () at foo.cpp:7

This is apparently due to certain pthreads functions not being included in the output executable.  This bug may duplicate #115157, and I apologize if so, but hopefully the included test case will be useful.

Additional info:

The suggestion in #115157 to forcibly link in all of libpthread.a is a valid workaround.

https://bugzilla.redhat.com/show_bug.cgi?id=115157 "executables linked statically with /usr/lib/nptl/libpthread.a fail" - 2004-2009 CLOSED WONTFIX

Jakub Jelinek 2004-10-29 05:26:10 EDT

First of all, avoid -static if you can, it only creates problems,
  both portability wise and others as well.

If you really need to create statically linked binary with -lpthread
  linked in, then just use -Wl,--whole-archive -lpthread -Wl,--no-whole-archive
  instead of -pthread.  Anything else has really many problems.


问:
Canonical way to do bulk create in django-rest-framework 3.x?
In 2.x we had a serializer which looked like:

```<code>class FooSerializer(serializers.ModelSerializer):
    bar = serializers.PrimaryKeyRelatedField()

    class Meta:
        model = Foo
</code>```

This effectively was able to handle bulk creates (passing a list as the body of a JSON post request). In 3.x, this endpoint is broken. I've tried to implement something similar to the docs on DRF

```<code>class FooListSerializer(serializers.ListSerializer):
    def create(self, validated_data):
        foos = [Foo(**item) for item in validated_data]
        return Foo.objects.bulk_create(foos)

class FooSerializer(serializers.ModelSerializer):
    bar = serializers.PrimaryKeyRelatedField(
        queryset=Bar.objects.all()
        )

    class Meta:
        model = Foo
        list_serializer_class = FooListSerializer
</code>```

And while this works for a single create request, when I attempt to pass a list I get the error:

```<code>AttributeError: 'FooListSerializer' object has no attribute 'object'
</code>```

I've seen some hacks where __init__ is super'd, but it seems with the creation of the ListSerializer class in 3.x there has to be a cleaner way to do this. Thanks in advance.

答:
You don't show how your code is making a FooSerializer instance. The Django REST Framework 3 documentation says:

To serialize a queryset or list of objects instead of a single object instance, you should pass the many=True flag when instantiating the serializer. You can then pass a queryset or list of objects to be serialized.

So, it seems that your code should detect whether the data contains one instance or many, and:

serializer = FooSerializer() to handle one instance, or

serializer = FooSerializer(many=True) to handle a list of many instances.

Explicit is better than implicit :-)

Body:
In 2.x we had a serializer which looked like:

class FooSerializer(serializers.ModelSerializer):
    bar = serializers.PrimaryKeyRelatedField()

    class Meta:
        model = Foo

This effectively was able to handle bulk creates (passing a list as the body of a JSON post request). In 3.x, this endpoint is broken. I've tried to implement something similar to the docs on DRF

class FooListSerializer(serializers.ListSerializer):
    def create(self, validated_data):
        foos = [Foo(**item) for item in validated_data]
        return Foo.objects.bulk_create(foos)

class FooSerializer(serializers.ModelSerializer):
    bar = serializers.PrimaryKeyRelatedField(
        queryset=Bar.objects.all()
        )

    class Meta:
        model = Foo
        list_serializer_class = FooListSerializer

And while this works for a single create request, when I attempt to pass a list I get the error:

AttributeError: 'FooListSerializer' object has no attribute 'object'

I've seen some hacks where __init__ is super'd, but it seems with the creation of the ListSerializer class in 3.x there has to be a cleaner way to do this. Thanks in advance.

ContentLicense: CC BY-SA 4.0


问:
What does "exited with code 9009" mean during this build?
What does this error message mean? What could I do to correct this issue?

AssemblyInfo.cs exited with code 9009

The problem is probably happening as part of a post-build step in a .NET solution in Visual Studio.

答:
Same as the other answers, in my case it was because of the missing file. To know what is the missing file, you can go to the output window and it will show you straight away what went missing.

To open the output window in Visual Studio:

Ctrl+Alt+O

View > Output

OwnerDisplayName: dgo


问:
how to pass a javascript var value to a Grails controller?
I have found the following javascript code to get browser window size and it works great!

```<code><script type="text/javascript">
<!--

 var viewportwidth;
 var viewportheight;

 // the more standards compliant browsers (mozilla/netscape/opera/IE7) use window.innerWidth and window.innerHeight

 if (typeof window.innerWidth != 'undefined')
 {
      viewportwidth = window.innerWidth,
      viewportheight = window.innerHeight
 }

// IE6 in standards compliant mode (i.e. with a valid doctype as the first line in the document)
 else if (typeof document.documentElement != 'undefined'
    && typeof document.documentElement.clientWidth !=
     'undefined' && document.documentElement.clientWidth != 0)
 {
       viewportwidth = document.documentElement.clientWidth,
       viewportheight = document.documentElement.clientHeight
 }

 // older versions of IE
 else
 {
       viewportwidth = document.body.clientWidth,
       viewportheight = document.body.clientHeight
 }
document.write('<p>Your viewport width is <b>'+viewportwidth+'x'+viewportheight+'</b>.</p>');
//-->
</script>
</code>```

Now I need to pass it to a Grails controller so I can resize a image according to screen size.

which is built using:

```<code><div align="center" valign="middle">
<img src="${createLink(controller:'chart', action:'buildChart')}" />
</div>
</code>```

How can I do so?
Thanks in advance!

答: Use javascript to add hidden form fields containing the numbers you want. From with in the grails controler use the request.getParameter("fieldName") to retrieve the values in the form of a string then convert to an int and do your resizing.</p>

Body:
I have found the following javascript code to get browser window size and it works great!

Now I need to pass it to a Grails controller so I can resize a image according to screen size.

which is built using:

How can I do so?
Thanks in advance!


问:
Optimizing memory usage for Resque
I have a Rails 4 app on Heroku with Resque and Resque Scheduler each running on separate 1x dynos. I currently have 3 Resque workers spun up.

Here is my Procfile for reference:

```<code>web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development}
resque: env TERM_CHILD=1 RESQUE_TERM_TIMEOUT=7 bundle exec rake resque:workers QUEUE='*' COUNT='3'
scheduler: bundle exec rake resque:scheduler
</code>```

When I profile the memory usage I can see that at idle, my resque dyno is consuming 390MB of 512MB of available memory, while my web dyno is consuming only 262MB of memory. At first glance this seems ok since I have plenty of head room.

When I queue up 40 emails to be sent out on the resque dyno though, memory usage spikes up to 480MB. If I queue more it grows until it exceeds the dyno's capacity and I get an R14 error. Note that I'm not sending the emails using the SendGrid API rather than through SMTP, but I still have to render_to_string each email template (looking into a more efficient route for that), which is what I suspect is consuming a lot of memory.

What I would like to know is what the best approach is here to optimize my resource consumption on Heroku? I use Resque mainly for image processing and sending batch emails. I currently have 3 workers spun up on one dyno, but I'm not sure if I should have more or fewer workers on one dyno or spread across multiple dynos. The current usage of the app is fairly low most of the time, however I expect the usage to increase greatly in the near future and am thus concerned about scaling. For example, right now I may have one user per day batch upload 100 photos at once, or send out 50 emails at once. I'm anticipating a 10 fold increase in the usage soon though.

I'm not looking for a specific answer per se, just advice on how to optimize my resource usage and scale up smoothly. Thanks!

答:
Heroku did a nice write-up on utilizing the resque-pool to reduce the memory overhead of resque workers.

The resque-pool gem scales out your resque based workers by running multiple workers per dyno, and is a good fit. The resque-pool gem behaves similarly to the Unicorn web server in the way it is designed. It boots one master process and then uses the unix fork command to create multiple resque workers that can all process your data independently. These multiple resque workers create your “pool” of workers. Using processes for concurrency is safer in Ruby but can have a higher memory overhead.

The usage of this gem also applies to running resque workers outside of the Heroku ecosystem as well.

Title: Optimizing memory usage for Resque

Body:
I have a Rails 4 app on Heroku with Resque and Resque Scheduler each running on separate 1x dynos. I currently have 3 Resque workers spun up.

Here is my Procfile for reference:

web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-development}
resque: env TERM_CHILD=1 RESQUE_TERM_TIMEOUT=7 bundle exec rake resque:workers QUEUE='*' COUNT='3'
scheduler: bundle exec rake resque:scheduler

When I profile the memory usage I can see that at idle, my resque dyno is consuming 390MB of 512MB of available memory, while my web dyno is consuming only 262MB of memory. At first glance this seems ok since I have plenty of head room.

When I queue up 40 emails to be sent out on the resque dyno though, memory usage spikes up to 480MB. If I queue more it grows until it exceeds the dyno's capacity and I get an R14 error. Note that I'm not sending the emails using the SendGrid API rather than through SMTP, but I still have to render_to_string each email template (looking into a more efficient route for that), which is what I suspect is consuming a lot of memory.

What I would like to know is what the best approach is here to optimize my resource consumption on Heroku? I use Resque mainly for image processing and sending batch emails. I currently have 3 workers spun up on one dyno, but I'm not sure if I should have more or fewer workers on one dyno or spread across multiple dynos. The current usage of the app is fairly low most of the time, however I expect the usage to increase greatly in the near future and am thus concerned about scaling. For example, right now I may have one user per day batch upload 100 photos at once, or send out 50 emails at once. I'm anticipating a 10 fold increase in the usage soon though.

I'm not looking for a specific answer per se, just advice on how to optimize my resource usage and scale up smoothly. Thanks!


问:
Iteratively Subset DataFrame based on Unique Times
Given the following example DataFrame:

```<code>>>> df
                 Times  Values
0  05/10/2017 01:01:03       1
1  05/10/2017 01:05:00       2
2  05/10/2017 01:06:10       3
3  05/11/2017 08:25:20       4
4  05/11/2017 08:30:14       5
5  05/11/2017 08:30:35       6
</code>```

I want to subset this DataFrame by the 'Time' column, by matching a partial string up to the hour.  For example, I want to subset using partial strings which contain "05/10/2017 01:" and "05/11/2017 08:" which breaks up the subsets into two new data frames:

```<code>>>> df1
                 Times  Values
0  05/10/2017 01:01:03       1
1  05/10/2017 01:05:00       2
2  05/10/2017 01:06:10       3
</code>```

and

```<code>>>> df2
0  05/11/2017 08:25:20       4
1  05/11/2017 08:30:14       5
2  05/11/2017 08:30:35       6
</code>```

Is it possible to make this subset iterative in Pandas, for multiple dates/times that similarly have the date/hour as the common identifier?

答:
You can use the str[] accessor to truncate the string representation of your date (you might have to cast astype(str) if your columns is a datetime and then use groupby.groups to access the dataframes as a dictionary where the keys are your truncated date values:

```<code>>>> df.groupby(df.Times.astype(str).str[0:13]).groups

{'2017-05-10 01': DatetimeIndex(['2017-05-10 01:01:03', '2017-05-10 01:05:00',
                '2017-05-10 01:06:10'],
               dtype='datetime64[ns]', name='time', freq=None),
 '2017-05-11 08': DatetimeIndex(['2017-05-11 08:25:20', '2017-05-11 08:30:14',
                '2017-05-11 08:30:35'],
               dtype='datetime64[ns]', name='time', freq=None)}
</code>```

Body:
Given the following example DataFrame:

>>> df
                 Times  Values
0  05/10/2017 01:01:03       1
1  05/10/2017 01:05:00       2
2  05/10/2017 01:06:10       3
3  05/11/2017 08:25:20       4
4  05/11/2017 08:30:14       5
5  05/11/2017 08:30:35       6

I want to subset this DataFrame by the 'Time' column, by matching a partial string up to the hour.  For example, I want to subset using partial strings which contain "05/10/2017 01:" and "05/11/2017 08:" which breaks up the subsets into two new data frames:

>>> df1
                 Times  Values
0  05/10/2017 01:01:03       1
1  05/10/2017 01:05:00       2
2  05/10/2017 01:06:10       3

and

>>> df2
0  05/11/2017 08:25:20       4
1  05/11/2017 08:30:14       5
2  05/11/2017 08:30:35       6

Is it possible to make this subset iterative in Pandas, for multiple dates/times that similarly have the date/hour as the common identifier?

Body:
You can use the str[] accessor to truncate the string representation of your date (you might have to cast astype(str) if your columns is a datetime and then use groupby.groups to access the dataframes as a dictionary where the keys are your truncated date values:

>>> df.groupby(df.Times.astype(str).str[0:13]).groups

{'2017-05-10 01': DatetimeIndex(['2017-05-10 01:01:03', '2017-05-10 01:05:00',
                '2017-05-10 01:06:10'],
               dtype='datetime64[ns]', name='time', freq=None),
 '2017-05-11 08': DatetimeIndex(['2017-05-11 08:25:20', '2017-05-11 08:30:14',
                '2017-05-11 08:30:35'],
               dtype='datetime64[ns]', name='time', freq=None)}


问:
Codeigniter and AJAX delete item from table leaves page blank
I managed to integrate ajax into my pagination and search. it seems to work fine so far. However, when I delete an item and try to refresh the list, my page just returns blank. I was hoping someone could guide me on the proper way to do this as I have tried various methods and want to try to use ajax. Thank you.

My view

```<code><script>
function searchFilter(page_num) {
    page_num = page_num?page_num:0;
    var keywords = $('#search').val();
        console.log(keywords);
        console.log('<?php echo site_url('/AdminDocuments/'); ?>'+page_num);
    $.ajax({
        type: 'POST',
        url: '<?php echo site_url('/AdminDocuments/Search/'); ?>'+page_num,
        data:'page='+page_num+'&keywords='+keywords,
        beforeSend: function () {
            $('.loading').show();
        },
        success: function (html) {
            $('#list').html(html);
            $('.loading').fadeOut("slow");
        }
    });
}

function deleteDocument(input){
    var id = input;
    console.log(id);
    $.ajax({
        type:'POST',
        url:'<?php echo site_url('/AdminDocuments/Delete'); ?>',
        data:'id='+id,
        beforeSend: function () {
            $('.loading').show();
        },
        success:function(result){
            console.log(result);
            $('#list').html(result);
            $('.loading').fadeOut("slow");
        }
    });
}
</script>
<div id="successid" style="display: none;" class="alert alert-success">
    <button type = "button" class = "close" data-dismiss = "alert" aria-hidden = "true">
        ×
    </button>
    Success
</div>
<div id="maincontainer">
    <div id="list" class="row">
        <div class="col-sm-12 table-responsive">
            <table class="table">
                <thead>
                    <tr class="text-left">
                        <td>Period</td>
                        <td>Name</td>
                        <td>Account Number</td>
                        <td></td>
                        <td></td>
                    </tr>
                </thead>
                <tbody>
                    <?php
                        if(!empty($datatable)){
                            foreach ($datatable as $data){
                    ?>
                        <tr class="text-left">
                            <td><?php echo $data['month']."/"; ?><?php echo $data['year']; ?></td>
                            <td><?php echo $data['first_name']; ?> <?php echo $data['last_name']; ?></td>
                            <td><?php echo $data['account_number']; ?></td>
                            <td><a href="<?php echo $data['file_link'];?>"><i class="fa fa-file" aria-hidden="true"></i> View</a></td>
                            <!--<td><a href="<?php echo site_url('AdminDocuments/Delete/'.$data['document_id']);?>"><i class="fa fa-trash" aria-hidden="true"></i> Delete</a></td>-->
                            <td><a href="#" id="delete" onclick="deleteDocument(<?php echo $data['document_id'];?>)"><i class="fa fa-trash" aria-hidden="true"></i> Delete</a></td>
                        </tr>
                    <?php
                        }
                    }else{
                    ?>
                        <tr>
                            <td colspan="7">You do not have any items at the moment</td>
                        </tr>
                    <?php
                    }
                    ?>
                </tbody>
            </table>
        </div>
        <div class="pagination-links pull-right">
            <?php echo $links; ?>
        </div>
    </div>
    <div class="loading" style="display: none;">
        <div class="content">
            <img src="<?php echo base_url().'assets/images/loading/loading.gif'; ?>"/>
        </div>
    </div>
</div>
</code>```

Controller

```<code>public function delete(){
        $id = $this->input->post('id');
        unlink($this->DocumentModel->admin_get($id)['file_location']);
        $result = $this->DocumentModel->delete($id);
        $activity = array(
            'page'=>'Documents',
            'action'=>'Deleted document',
            'created_on'=>date('Y-m-d'),
            'time_on'=>date('h:i:s')
        );
        $this->ActivitiesModel->admin_create($activity);
        if($result===false){
            echo "false";
        }else{
            echo "true";
        }
    }
</code>```

答:
This line in the success of  the deleteDocument function $('#list').html(result);  is causing the #list div to be replaced by either true or false
 What you're looking for is to call the pagination script again on success</p>

Body:
I managed to integrate ajax into my pagination and search. it seems to work fine so far. However, when I delete an item and try to refresh the list, my page just returns blank. I was hoping someone could guide me on the proper way to do this as I have tried various methods and want to try to use ajax. Thank you.

My view

    <button type = "button" class = "close" data-dismiss = "alert" aria-hidden = "true">
        ×
    </button>
    Success

                        Period
                        Name
                        Account Number

                    <?php
                        if(!empty($datatable)){
                            foreach ($datatable as $data){
                    ?>

                            <?php echo $data['month']."/"; ?><?php echo $data['year']; ?>
                            <?php echo $data['first_name']; ?> <?php echo $data['last_name']; ?>
                            <?php echo $data['account_number']; ?>
                            <a href="<?php echo $data['file_link'];?>"> View

                            <a href="#" id="delete" onclick="deleteDocument(<?php echo $data['document_id'];?>)"> Delete

                    <?php
                        }
                    }else{
                    ?>

                            You do not have any items at the moment

                    <?php
                    }
                    ?>

            <?php echo $links; ?>

            <img src="<?php echo base_url().'assets/images/loading/loading.gif'; ?>"/>

Controller

public function delete(){
        $id = $this->input->post('id');
        unlink($this->DocumentModel->admin_get($id)['file_location']);
        $result = $this->DocumentModel->delete($id);
        $activity = array(
            'page'=>'Documents',
            'action'=>'Deleted document',
            'created_on'=>date('Y-m-d'),
            'time_on'=>date('h:i:s')
        );
        $this->ActivitiesModel->admin_create($activity);
        if($result===false){
            echo "false";
        }else{
            echo "true";
        }
    }


问:
How to prevent flexbox stretching my image horizontally?
I saw this question about flexbox stretching images vertically, and applying align-self: center fixes it:
Why does flexbox stretch my image?

However, I have an issues with my images stretched horizontally:
https://jsfiddle.net/qnnpxskk/

HTML:

```<code><div class="wrapper">
  <div class="inner">
    <div class="row">
      <img src="https://www.wired.com/wp-content/uploads/2015/11/google-tensor-flow-logo-F.jpg">
    </div>
    <div class="row">
      <img src="https://www.wired.com/wp-content/uploads/2015/11/google-tensor-flow-logo-F.jpg">
    </div>
  </div>
</div>
</code>```

CSS:

```<code>.wrapper{
  width: 600px;
  height: 900px;
  background: white;
}
.inner{
  display: flex;
  flex-direction: column;
}
.row{
  max-height: 100px;
  margin: 20px;
  display: flex;
  flex-direction: row;
  background: red;
}
.row img{
  max-height: 100%;
}
</code>```

Everything I try I can't seem to get the image to not be stretched. I want the image to be 100% height of the parent and the width to maintain aspect ratio of the original.

答:
Update css part

```<code>.row img{
  max-height: 100%;
  object-fit:contain; /* Add this You can change it as your need */
}
.inner{
  display: flex;
  flex-flow:column nowrap;
  justify-content:flex-start;
}
.wrapper{
  width: 100%; /*Add this*/
  height: 900px;
  background: white;
}
</code>```

update fiddle

Update fiddle

.wrapper{
  width: 100%;
  height: 900px;
  background: white;
}
.inner{
  display: flex;
  flex-flow:column nowrap;
  justify-content:flex-start;
}
.row{
  max-height: 100px;
  margin: 20px;
  display: flex;
  flex-direction: row;
  background: red;
}
.row img{
  max-height: 100%;
  object-fit:contain;
}```
<pre class="snippet-code-html lang-html prettyprint-override"><code><div class="wrapper">
  <div class="inner">
    <div class="row">
      <img src="https://www.wired.com/wp-content/uploads/2015/11/google-tensor-flow-logo-F.jpg">
    </div>
    <div class="row">
      <img src="https://www.wired.com/wp-content/uploads/2015/11/google-tensor-flow-logo-F.jpg">
    </div>
  </div>
</div></code>```

About object-fit https://css-tricks.com/almanac/properties/o/object-fit/

Body:
I saw this question about flexbox stretching images vertically, and applying align-self: center fixes it:
Why does flexbox stretch my image?

However, I have an issues with my images stretched horizontally:
https://jsfiddle.net/qnnpxskk/

HTML:

CSS:

.wrapper{
  width: 600px;
  height: 900px;
  background: white;
}
.inner{
  display: flex;
  flex-direction: column;
}
.row{
  max-height: 100px;
  margin: 20px;
  display: flex;
  flex-direction: row;
  background: red;
}
.row img{
  max-height: 100%;
}

Everything I try I can't seem to get the image to not be stretched. I want the image to be 100% height of the parent and the width to maintain aspect ratio of the original.

Body:
Update css part

.row img{
  max-height: 100%;
  object-fit:contain; /* Add this You can change it as your need */
}
.inner{
  display: flex;
  flex-flow:column nowrap;
  justify-content:flex-start;
}
.wrapper{
  width: 100%; /*Add this*/
  height: 900px;
  background: white;
}

update fiddle

Update fiddle

.wrapper{
  width: 100%;
  height: 900px;
  background: white;
}
.inner{
  display: flex;
  flex-flow:column nowrap;
  justify-content:flex-start;
}
.row{
  max-height: 100px;
  margin: 20px;
  display: flex;
  flex-direction: row;
  background: red;
}
.row img{
  max-height: 100%;
  object-fit:contain;
}

<div class="wrapper">
  <div class="inner">
    <div class="row">
      <img src="https://www.wired.com/wp-content/uploads/2015/11/google-tensor-flow-logo-F.jpg">
    </div>
    <div class="row">
      <img src="https://www.wired.com/wp-content/uploads/2015/11/google-tensor-flow-logo-F.jpg">
    </div>
  </div>
</div>

About object-fit https://css-tricks.com/almanac/properties/o/object-fit/


问:
How do I embed a JFrame into a JavaFX 2.0 application?
I have searched stackoverflow extensively for help on this topic, but the Q&As I found are old and the answers have changed for the current version of the JDK (I'm currently using 7u51).

Note that I was never SUPER proficient in Swing to begin with, but I believe I understand the fundamentals. (I've always been more focused on the meat inside an app, not the GUI).

I'm trying to work with a third party library. The third party library requires that it's components use JFrame.

Therefore, I'm trying to see how I would embed a JFrame into my JavaFX application. There was an old answer about doing something with javafx.ext.swing, but that's no longer included in JavaFX.

Help?

==========

I should also add: I think Java 8, which is currently beta, will support what I need based on this: http://docs.oracle.com/javafx/8/embed_swing/jfxpub-embed_swing.htm , but I need to see if there is a way to do this without relying on a beta product.

答:
Create a SwingNode s

Create a JComponent c

Call s.setContent(c)


问:
Polynomial size CFG such that each terminal in a word occurs even number of times (large alphabet)
Find a context-free grammar (CFG) for the language L of all
words such that each terminal in a word occurs even number of times over a possibly large alphabet Σ

My long aproach is (the only nonterminal is S):

S ⟶ ε | SS

x ∈ Σ : S ⟶ xSx

x,y ∈ Σ : S ⟶ xxSyy | yySxx | xySxy | xySyx | yxSyx | yxSyx

Is this correct?
The productions generate correct words, do they generate all words?

EDIT: Can a CFG over large alphabet describe a language, where each terminal appears even number of times?

EDIT_2: If a solution exists, is it possible Chomsky Normal Form be polynomial in |Σ| ?

答:
There is even a regular grammar to achieve this. Since every regular grammar is by definition context-free, the answer ist "yes". It's also possible to construct a finite automaton, but since you asked for a grammar...

Here's how: recall that a regular grammar allows rules of the form A -> b C or A -> b or A -> epsilon, where A and C are non-terminals and b is a terminal. This essentially means that every non-terminal generates a terminal and another non-terminal which will generate the rest of the string; we could say that every non-terminal encodes a certain property about the strings it generates.

Consider now all the subsets of  the alphabet Sigma. Since Sigma is supposed to be finite, so is the set of subsets (powerset). Let the set of non-terminals be the powerset of Sigma.

Start with a rule: {} -> a {a} for every terminal a.
For every non-terminal X, add the rule X -> a X-{a} if a is in X; or X -> a X+{a} if a is not in X
(I'm sloppily writing + and - for set union and difference here).

Finally, add {} -> epsilon (the empty word).

The grammar encodes in its non-terminals exactly the sets of terminals which have appeared in an odd number and thus have to be "canceled out" again.


问:
Android Runtime.getRuntime().exec("su") java.io.IOException: Permission denied
When I use this code:

```<code>process = Runtime.getRuntime().exec("su");
os = new DataOutputStream(process.getOutputStream());
</code>```

there is always an error, any ideas? Running on an android virtual device

```<code>W/System.err(3211): java.io.IOException: Error running exec(). Command: [su] Working Directory: null Environment: null
W/System.err(3211):     at java.lang.ProcessManager.exec(ProcessManager.java:211)
W/System.err(3211):     at java.lang.Runtime.exec(Runtime.java:174)
W/System.err(3211):     at java.lang.Runtime.exec(Runtime.java:247)
W/System.err(3211):     at java.lang.Runtime.exec(Runtime.java:190)
W/System.err(3211):     at com.example.executeandroidtest.ShellUtils.execCommand(MainActivity.java:661)
W/System.err(3211):     at com.example.executeandroidtest.MainActivity$3.run(MainActivity.java:410)
W/System.err(3211):     at java.lang.Thread.run(Thread.java:818)
W/System.err(3211): Caused by: java.io.IOException: Permission denied
W/System.err(3211):     at java.lang.ProcessManager.exec(Native Method)
W/System.err(3211):     at java.lang.ProcessManager.exec(ProcessManager.java:209)
W/System.err(3211):     ... 6 more
</code>```

答:
Caused by: java.io.IOException: Permission denied

find where su is, for instance: /system/xbin/su

check su's file permission: ls -al /system/xbin/su

change mod if necessary: chmod 4755 su

try again.

Body:
When I use this code:

process = Runtime.getRuntime().exec("su");
os = new DataOutputStream(process.getOutputStream());

there is always an error, any ideas? Running on an android virtual device

W/System.err(3211): java.io.IOException: Error running exec(). Command: [su] Working Directory: null Environment: null
W/System.err(3211):     at java.lang.ProcessManager.exec(ProcessManager.java:211)
W/System.err(3211):     at java.lang.Runtime.exec(Runtime.java:174)
W/System.err(3211):     at java.lang.Runtime.exec(Runtime.java:247)
W/System.err(3211):     at java.lang.Runtime.exec(Runtime.java:190)
W/System.err(3211):     at com.example.executeandroidtest.ShellUtils.execCommand(MainActivity.java:661)
W/System.err(3211):     at com.example.executeandroidtest.MainActivity$3.run(MainActivity.java:410)
W/System.err(3211):     at java.lang.Thread.run(Thread.java:818)
W/System.err(3211): Caused by: java.io.IOException: Permission denied
W/System.err(3211):     at java.lang.ProcessManager.exec(Native Method)
W/System.err(3211):     at java.lang.ProcessManager.exec(ProcessManager.java:209)
W/System.err(3211):     ... 6 more


问:
How to set text from bundle (toolbar)
How to set text from bundle (toolbar) I have no idea how to set it.

toolbar.xml

```<code><android.support.v7.widget.Toolbar xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:id="@+id/my_toolbar"
    android:layout_width="match_parent"
    android:layout_height="wrap_content"
    android:background="?attr/colorPrimary"
    android:minHeight="?attr/actionBarSize"
    app:theme="@style/ToolbarColor">

    <TextView
        android:id="@+id/textView_toolbar"
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:text="My example"
        android:gravity="center"/>
</android.support.v7.widget.Toolbar>
</code>```

DetailActivity.java

```<code>Bundle bundle = getIntent().getExtras();
        if (bundle!=null){
            String name = bundle.getString("nameCar");
        }

   Toolbar toolbar = (Toolbar) findViewById(R.id.m_tool_bar_detail);
        TextView textToolbar = (TextView) toolbar.findViewById(R.id.textView_toolbar);
        setSupportActionBar(toolbar);
        if(getSupportActionBar()!=null){

            textToolbar.setText(name); ///ERROR(name)
            getSupportActionBar().setIcon(getDrawable(R.drawable.ic_action_name));
        }
</code>```

I've tried textToolbar.setText("xxxxx"); but it gives an error.

I want to get name in bundle to set Textview in toolbar.

答:
The name variable is inaccessible outside the if block and you should be getting a warning in there. Move that variable outside the if block and you should be fine

```<code>    Bundle bundle = getIntent().getExtras();
    String name = null;

    if (bundle != null){
        name = bundle.getString("nameCar");
    }
</code>```

Title: How to set text from bundle (toolbar)

Body:
How to set text from bundle (toolbar) I have no idea how to set it.

toolbar.xml

<android.support.v7.widget.Toolbar xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:id="@+id/my_toolbar"
    android:layout_width="match_parent"
    android:layout_height="wrap_content"
    android:background="?attr/colorPrimary"
    android:minHeight="?attr/actionBarSize"
    app:theme="@style/ToolbarColor">

    <TextView
        android:id="@+id/textView_toolbar"
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:text="My example"
        android:gravity="center"/>
</android.support.v7.widget.Toolbar>

DetailActivity.java

Bundle bundle = getIntent().getExtras();
        if (bundle!=null){
            String name = bundle.getString("nameCar");
        }

   Toolbar toolbar = (Toolbar) findViewById(R.id.m_tool_bar_detail);
        TextView textToolbar = (TextView) toolbar.findViewById(R.id.textView_toolbar);
        setSupportActionBar(toolbar);
        if(getSupportActionBar()!=null){

            textToolbar.setText(name); ///ERROR(name)
            getSupportActionBar().setIcon(getDrawable(R.drawable.ic_action_name));
        }

I've tried textToolbar.setText("xxxxx"); but it gives an error.

I want to get name in bundle to set Textview in toolbar.

Body:
The name variable is inaccessible outside the if block and you should be getting a warning in there. Move that variable outside the if block and you should be fine

    Bundle bundle = getIntent().getExtras();
    String name = null;

    if (bundle != null){
        name = bundle.getString("nameCar");
    }


问:
Submitting HTML array to jQuery AJAX
I have an HTML array and I want to submit it via jQuery ajax. When submitting and testing directly in the browser (just submitting the form as usual) I get the expected results and everything works. However, when submitting via ajax I get empty arrays.

I've tried serializing the form as well, but that produces the same result. Any thoughts as to what I'm doing wrong?

My form

```<code><form id="form-delete-previews" action="/delete/media/" method="DELETE">
@foreach($club->media as $image)
    <div class="col-xs-3">
        <input type="checkbox" id="cb{{ $loop->iteration }}" name="image[]" value="{{ $image->id }}">
        <label for="cb{{ $loop->iteration }}">
            <img src="{{ $image->getUrl('sm') }}" class="img-responsive img-rounded" alt="">
        </label>
    </div>
@endforeach
.....
</code>```

My javascript

```<code>$('#form-delete-previews').submit(function(e) {
    e.preventDefault();
    var form = $(this);
    formdata = new FormData(form[0]);
    swal({
        title: 'Are You Sure?',
        text: 'This can not be undone!',
        type: 'error',
        showCancelButton: true,
        confirmButtonColor: '#ee5254',
        confirmButtonText: 'Delete Images',
    }).then(function() {
        $.ajax({
            type: 'DELETE',
            url: form.attr('action'),
            headers: { 'X-CSRF-TOKEN': token },
            data: formdata,
            cache: false,
            contentType: false,
            processData: false,
            dataType: 'json',
            success: function(response) {
                success();
            },
            error: function(data) {
                oops();
            }
        });
    });
});
</code>```

My Laravel Controller

```<code>public function destroyMedia(Request $request)
{
    // return here for testing
    return response()->json([
        'type' => $request,
    ]);

    .....
}
</code>```

This is what's returned

```<code>{
    "type": {
        "attributes": {},
        "request": {},
        "query": {},
        "server": {},
        "files": {},
        "cookies": {},
        "headers": {}
    }
}
</code>```

UPDATE

Not sure why, but if I change the method from DELETE to POST it works.

Any idea why DELETE wouldn't work?

答:
You are serializing the Request object in the controller. Instead, try:

```<code>// return here for testing
return response()->json([
    'type' => $request->get('image'),
]);
</code>```

and see what is returned. Also, I would do this:

```<code>formdata = new FormData(form[0]);
console.log(formdata);
</code>```

to see what is actually being sent.

Title: Submitting HTML array to jQuery AJAX

Body:
I have an HTML array and I want to submit it via jQuery ajax. When submitting and testing directly in the browser (just submitting the form as usual) I get the expected results and everything works. However, when submitting via ajax I get empty arrays.

I've tried serializing the form as well, but that produces the same result. Any thoughts as to what I'm doing wrong?

My form

<form id="form-delete-previews" action="/delete/media/" method="DELETE">
@foreach($club->media as $image)

        <input type="checkbox" id="cb{{ $loop->iteration }}" name="image[]" value="{{ $image->id }}">
        iteration }}">
            getUrl('sm') }}" class="img-responsive img-rounded" alt="">

@endforeach
.....

My javascript

$('#form-delete-previews').submit(function(e) {
    e.preventDefault();
    var form = $(this);
    formdata = new FormData(form[0]);
    swal({
        title: 'Are You Sure?',
        text: 'This can not be undone!',
        type: 'error',
        showCancelButton: true,
        confirmButtonColor: '#ee5254',
        confirmButtonText: 'Delete Images',
    }).then(function() {
        $.ajax({
            type: 'DELETE',
            url: form.attr('action'),
            headers: { 'X-CSRF-TOKEN': token },
            data: formdata,
            cache: false,
            contentType: false,
            processData: false,
            dataType: 'json',
            success: function(response) {
                success();
            },
            error: function(data) {
                oops();
            }
        });
    });
});

My Laravel Controller

public function destroyMedia(Request $request)
{
    // return here for testing
    return response()->json([
        'type' => $request,
    ]);

    .....
}

This is what's returned

{
    "type": {
        "attributes": {},
        "request": {},
        "query": {},
        "server": {},
        "files": {},
        "cookies": {},
        "headers": {}
    }
}

UPDATE

Not sure why, but if I change the method from DELETE to POST it works.

Any idea why DELETE wouldn't work?

Body:
You are serializing the Request object in the controller. Instead, try:

// return here for testing
return response()->json([
    'type' => $request->get('image'),
]);

and see what is returned. Also, I would do this:

formdata = new FormData(form[0]);
console.log(formdata);

to see what is actually being sent.


问:
Securing files and folders with htaccess
I have a couple of files on my server that contains sensitive information. Only the server should be allowed to edit these files, no one else should be able to read/access them. They are stored as .txt.

I've stored them in a separate folder, and added a .htaccess file with:

```<code><Files *>
Deny from all
</Files>
</code>```

My question is weather it's secure enough to store sensitive information with .htaccess, or if someone can hack it and get access to the files?

Thanks

答:
.htaccess is as secure as you can get, on a server-side basis.

All .ht files by default are un-accessible to the public, so no-one can edit or view the .htaccess file unless they access it through FTP ect. So the .htaccess file is secure as your server is.

Body:
I have a couple of files on my server that contains sensitive information. Only the server should be allowed to edit these files, no one else should be able to read/access them. They are stored as .txt.

I've stored them in a separate folder, and added a .htaccess file with:

<Files *>
Deny from all
</Files>

My question is weather it's secure enough to store sensitive information with .htaccess, or if someone can hack it and get access to the files?

Thanks


问:
Locate mismatch position of regular expression
I'm currently using python to analyze the log lines from a huge log file, and I use the regular expression to find out those lines of interest.

However, how could I tell certain position that doesnt fit in the regex? I cannot do this manually, 'cause it's too painful.

Take the following two log lines as an example, I don't understand why logline doesnt match with the given standard while logline2 does. Any idea how I can find out?

```<code>>>> reg_mld_str='.*Contract=(?P<feedcode>\S+).*LastPrice=(?P<lastprice>\S+) MatchTotQty=(?P<turnover_vol>\d+) Turnover=(?P<turnover_val>\d+) RiseLimit=(?P<riselimit>\S+) FallLimit=(?P<falllimit>\S+).* BuyPriceOne=(?P<lv1_price_bid>\S+) BuyQtyOne=(?P<lv1_vol_bid>\d+) BuyImplyQtyOne=(?P<lv1_imply_bid>\d+) BuyPriceTwo=(?P<lv2_price_bid>\S+) BuyQtyTwo=(?P<lv2_vol_bid>\d+) BuyImplyQtyTwo=(?P<lv2_imply_bid>\d+) BuyPriceThree=(?P<lv3_price_bid>\S+) BuyQtyThree=(?P<lv3_vol_bid>\d+) BuyImplyQtyThree=(?P<lv3_imply_bid>\d+) BuyPriceFour=(?P<lv4_price_bid>\S+) BuyQtyFour=(?P<lv4_vol_bid>\d+) BuyImplyQtyFour=(?P<lv4_imply_bid>\d+) BuyPriceFive=(?P<lv5_price_bid>\S+) BuyQtyFive=(?P<lv5_vol_bid>\d+) BuyImplyQtyFive=(?P<lv5_imply_bid>\d+) SellPriceOne=(?P<lv1_price_ask>\S+) SellQtyOne=(?P<lv1_vol_ask>\d+) SellImplyQtyOne=(?P<lv1_imply_ask>\d+) SellPriceTwo=(?P<lv2_price_ask>\S+) SellQtyTwo=(?P<lv2_vol_ask>\d+) SellImplyQtyTwo=(?P<lv2_imply_ask>\d+) SellPriceThree=(?P<lv3_price_ask>\S+) SellQtyThree=(?P<lv3_vol_ask>\d+) SellImplyQtyThree=(?P<lv3_imply_ask>\d+) SellPriceFour=(?P<lv4_price_ask>\S+) SellQtyFour=(?P<lv4_vol_ask>\d+) SellImplyQtyFour=(?P<lv4_imply_ask>\d+) SellPriceFive=(?P<lv5_price_ask>\S+) SellQtyFive=(?P<lv5_vol_ask>\d+) SellImplyQtyFive=(?P<lv5_imply_ask>\d+).*GenTime=(?P<ts>\S+) LastMatchQty'
>>> reg_mld=re.compile(reg_mld_str)
>>> logline2= '15:19:25.413628808 [Debug ] [LEVEL2_SESSION] [anonymous] [OnBestAndDeep] Info="Level2 PriceDepth." BestAndDeep={Type=1 Length=488 Version=0 Time=0 Exchange= Contract=m1712-P-2550 SuspensionSign=0 LastClearPrice=70.5 ClearPrice=70.5 AvgPrice=0.5 LastClose=70.5 Close=0.5 OpenPrice=0.5 LastOpenInterest=172 OpenInterest=172 LastPrice=0.5 MatchTotQty=2 Turnover=10 RiseLimit=212.5 FallLimit=0.5 HighPrice=0.5 LowPrice=0.5 PreDelta=0 CurrDelta=0 BuyPriceOne=0 BuyQtyOne=0 BuyImplyQtyOne=0 BuyPriceTwo=0 BuyQtyTwo=0 BuyImplyQtyTwo=0 BuyPriceThree=0 BuyQtyThree=0 BuyImplyQtyThree=0 BuyPriceFour=0 BuyQtyFour=0 BuyImplyQtyFour=0 BuyPriceFive=0 BuyQtyFive=0 BuyImplyQtyFive=0 SellPriceOne=0 SellQtyOne=0 SellImplyQtyOne=0 SellPriceTwo=0 SellQtyTwo=0 SellImplyQtyTwo=0 SellPriceThree=0 SellQtyThree=0 SellImplyQtyThree=0 SellPriceFour=0 SellQtyFour=0 SellImplyQtyFour=0 SellPriceFive=0 SellQtyFive=0 SellImplyQtyFive=0 GenTime=15:19:09.609 LastMatchQty=2 InterestChg=0 LifeLow=0.5 LifeHigh=284 Delta=-0.2256374694 Gamma=0.0006061540941 Rho=-0.3017551265 Theta=-0.3659621086 Vega=6.354866111 TradeDate=20170420 LocalDate=}'
>>> reg_mld.match(logline2)
<_sre.SRE_Match object at 0x0000000002322BC0>
>>> logline = '09:26:00.632018658 [Debug ] [LEVEL2_SESSION] [anonymous] [OnBestAndDeep] Info="Level2 PriceDepth." BestAndDeep={Type=1 Length=388 Version=0 Time=30813619 Exchange= Contract=i1709 SuspensionSign=0 LastClearPrice=426.5 ClearPrice=inf AvgPrice=436 LastClose=429.5 Close=inf OpenPrice=427 LastOpenInterest=2088194 OpenInterest=1915258 LastPrice=437.5 MatchTotQty=1935046 Turnover=8.44126785e+10 RiseLimit=460.5 FallLimit=392.5 HighPrice=443 LowPrice=422.5 PreDelta=0 CurrDelta=0 BuyPriceOne=437.5 BuyQtyOne=1122 BuyImplyQtyOne=12 BuyPriceTwo=437 BuyQtyTwo=1683 BuyImplyQtyTwo=175 BuyPriceThree=436.5 BuyQtyThree=1647 BuyImplyQtyThree=0 BuyPriceFour=436 BuyQtyFour=1388 BuyImplyQtyFour=0 BuyPriceFive=435.5 BuyQtyFive=1499 BuyImplyQtyFive=0 SellPriceOne=438 SellQtyOne=1119 SellImplyQtyOne=4 SellPriceTwo=438.5 SellQtyTwo=3414 SellImplyQtyTwo=0 SellPriceThree=439 SellQtyThree=2773 SellImplyQtyThree=1 SellPriceFour=439.5 SellQtyFour=2338 SellImplyQtyFour=0 SellPriceFive=440 SellQtyFive=3440 SellImplyQtyFive=0 GenTime=09:26:01.012 LastMatchQty=4 InterestChg=-172936 LifeLow=343.5 LifeHigh=685 Delta=0 Gamma=0 Rho=0 Theta=0 Vega=0 TradeDate=20170605 LocalDate=}'
>>> reg_mld.match(logline)
</code>```

答:
The problem is on

Turnover=8.44126785e+10

if I replace that for something like:

Turnover=84412678510

Your regex will match:
https://regex101.com/r/vH6l5M/2

Body:
I'm currently using python to analyze the log lines from a huge log file, and I use the regular expression to find out those lines of interest.

However, how could I tell certain position that doesnt fit in the regex? I cannot do this manually, 'cause it's too painful.

Take the following two log lines as an example, I don't understand why logline doesnt match with the given standard while logline2 does. Any idea how I can find out?

>>> reg_mld_str='.*Contract=(?P<feedcode>\S+).*LastPrice=(?P<lastprice>\S+) MatchTotQty=(?P<turnover_vol>\d+) Turnover=(?P<turnover_val>\d+) RiseLimit=(?P<riselimit>\S+) FallLimit=(?P<falllimit>\S+).* BuyPriceOne=(?P<lv1_price_bid>\S+) BuyQtyOne=(?P<lv1_vol_bid>\d+) BuyImplyQtyOne=(?P<lv1_imply_bid>\d+) BuyPriceTwo=(?P<lv2_price_bid>\S+) BuyQtyTwo=(?P<lv2_vol_bid>\d+) BuyImplyQtyTwo=(?P<lv2_imply_bid>\d+) BuyPriceThree=(?P<lv3_price_bid>\S+) BuyQtyThree=(?P<lv3_vol_bid>\d+) BuyImplyQtyThree=(?P<lv3_imply_bid>\d+) BuyPriceFour=(?P<lv4_price_bid>\S+) BuyQtyFour=(?P<lv4_vol_bid>\d+) BuyImplyQtyFour=(?P<lv4_imply_bid>\d+) BuyPriceFive=(?P<lv5_price_bid>\S+) BuyQtyFive=(?P<lv5_vol_bid>\d+) BuyImplyQtyFive=(?P<lv5_imply_bid>\d+) SellPriceOne=(?P<lv1_price_ask>\S+) SellQtyOne=(?P<lv1_vol_ask>\d+) SellImplyQtyOne=(?P<lv1_imply_ask>\d+) SellPriceTwo=(?P<lv2_price_ask>\S+) SellQtyTwo=(?P<lv2_vol_ask>\d+) SellImplyQtyTwo=(?P<lv2_imply_ask>\d+) SellPriceThree=(?P<lv3_price_ask>\S+) SellQtyThree=(?P<lv3_vol_ask>\d+) SellImplyQtyThree=(?P<lv3_imply_ask>\d+) SellPriceFour=(?P<lv4_price_ask>\S+) SellQtyFour=(?P<lv4_vol_ask>\d+) SellImplyQtyFour=(?P<lv4_imply_ask>\d+) SellPriceFive=(?P<lv5_price_ask>\S+) SellQtyFive=(?P<lv5_vol_ask>\d+) SellImplyQtyFive=(?P<lv5_imply_ask>\d+).*GenTime=(?P<ts>\S+) LastMatchQty'
>>> reg_mld=re.compile(reg_mld_str)
>>> logline2= '15:19:25.413628808 [Debug ] [LEVEL2_SESSION] [anonymous] [OnBestAndDeep] Info="Level2 PriceDepth." BestAndDeep={Type=1 Length=488 Version=0 Time=0 Exchange= Contract=m1712-P-2550 SuspensionSign=0 LastClearPrice=70.5 ClearPrice=70.5 AvgPrice=0.5 LastClose=70.5 Close=0.5 OpenPrice=0.5 LastOpenInterest=172 OpenInterest=172 LastPrice=0.5 MatchTotQty=2 Turnover=10 RiseLimit=212.5 FallLimit=0.5 HighPrice=0.5 LowPrice=0.5 PreDelta=0 CurrDelta=0 BuyPriceOne=0 BuyQtyOne=0 BuyImplyQtyOne=0 BuyPriceTwo=0 BuyQtyTwo=0 BuyImplyQtyTwo=0 BuyPriceThree=0 BuyQtyThree=0 BuyImplyQtyThree=0 BuyPriceFour=0 BuyQtyFour=0 BuyImplyQtyFour=0 BuyPriceFive=0 BuyQtyFive=0 BuyImplyQtyFive=0 SellPriceOne=0 SellQtyOne=0 SellImplyQtyOne=0 SellPriceTwo=0 SellQtyTwo=0 SellImplyQtyTwo=0 SellPriceThree=0 SellQtyThree=0 SellImplyQtyThree=0 SellPriceFour=0 SellQtyFour=0 SellImplyQtyFour=0 SellPriceFive=0 SellQtyFive=0 SellImplyQtyFive=0 GenTime=15:19:09.609 LastMatchQty=2 InterestChg=0 LifeLow=0.5 LifeHigh=284 Delta=-0.2256374694 Gamma=0.0006061540941 Rho=-0.3017551265 Theta=-0.3659621086 Vega=6.354866111 TradeDate=20170420 LocalDate=}'
>>> reg_mld.match(logline2)
<_sre.SRE_Match object at 0x0000000002322BC0>
>>> logline = '09:26:00.632018658 [Debug ] [LEVEL2_SESSION] [anonymous] [OnBestAndDeep] Info="Level2 PriceDepth." BestAndDeep={Type=1 Length=388 Version=0 Time=30813619 Exchange= Contract=i1709 SuspensionSign=0 LastClearPrice=426.5 ClearPrice=inf AvgPrice=436 LastClose=429.5 Close=inf OpenPrice=427 LastOpenInterest=2088194 OpenInterest=1915258 LastPrice=437.5 MatchTotQty=1935046 Turnover=8.44126785e+10 RiseLimit=460.5 FallLimit=392.5 HighPrice=443 LowPrice=422.5 PreDelta=0 CurrDelta=0 BuyPriceOne=437.5 BuyQtyOne=1122 BuyImplyQtyOne=12 BuyPriceTwo=437 BuyQtyTwo=1683 BuyImplyQtyTwo=175 BuyPriceThree=436.5 BuyQtyThree=1647 BuyImplyQtyThree=0 BuyPriceFour=436 BuyQtyFour=1388 BuyImplyQtyFour=0 BuyPriceFive=435.5 BuyQtyFive=1499 BuyImplyQtyFive=0 SellPriceOne=438 SellQtyOne=1119 SellImplyQtyOne=4 SellPriceTwo=438.5 SellQtyTwo=3414 SellImplyQtyTwo=0 SellPriceThree=439 SellQtyThree=2773 SellImplyQtyThree=1 SellPriceFour=439.5 SellQtyFour=2338 SellImplyQtyFour=0 SellPriceFive=440 SellQtyFive=3440 SellImplyQtyFive=0 GenTime=09:26:01.012 LastMatchQty=4 InterestChg=-172936 LifeLow=343.5 LifeHigh=685 Delta=0 Gamma=0 Rho=0 Theta=0 Vega=0 TradeDate=20170605 LocalDate=}'
>>> reg_mld.match(logline)


问:
Merging two multi-dimensional arrays using key and adding values
I want to merge two array's using key(product_id) and adding that values(usage).
Array 1

```
Array
(
    [0] => Array
        (
            [name] => Reschedule A Service
            [usage] => 1
            [product_id] => 8
        )

    [1] => Array
        (
            [name] => Adding An Image
            [usage] => 1
            [product_id] => 5
        )

    [2] => Array
        (
            [name] => Each Calendar Event
            [usage] => 1
            [product_id] => 14
        )

)
```

Array 2

```
Array
(
    [0] => Array
        (
            [name] => Adding An Image
            [usage] => 1
            [product_id] => 5
        )

    [1] => Array
        (
            [name] => Schedule A Service
            [usage] => 3
            [product_id] => 11
        )

    [2] => Array
        (
            [name] => Each Calendar Event
            [usage] => 2
            [product_id] => 14
        )

    [3] => Array
        (
            [name] => Sales Performance Dashboard
            [usage] => 2
            [product_id] => 30
        )

    [4] => Array
        (
            [name] => Quote
            [usage] => 1
            [product_id] => 32
        )

)
```

I need an out put like this merging and adding usage values.

```
Array
(
    [0] => Array
        (
            [name] => Adding An Image
            [usage] => 2
            [product_id] => 5
        )

    [1] => Array
        (
            [name] => Schedule A Service
            [usage] => 3
            [product_id] => 11
        )

    [2] => Array
        (
            [name] => Each Calendar Event
            [usage] => 3
            [product_id] => 14
        )

    [3] => Array
        (
            [name] => Sales Performance Dashboard
            [usage] => 2
            [product_id] => 30
        )

    [4] => Array
        (
            [name] => Quote
            [usage] => 1
            [product_id] => 32
        )

    [5] => Array
        (
            [name] => Reschedule A Service
            [usage] => 1
            [product_id] => 8
        )

)
```

This is my code for creating arrays

```

    foreach($query->rows as $product){

     $top_products[]=array(
        'name'=>$product['name'],
        'usage'=>$product['pusage'],
        'product_id'=>$product['product_id']
     );

    }

```

```

    foreach($query_2->rows as $product){

         $top_point_products[]=array(
           'name'=>$product['name'],
           'usage'=>$product['p_usage'],
           'product_id'=>$product['product_id']
         );

    }

```

答:
Use array_merge and a simple foreach loop to check your condition and update the usagevalues.

See below

```<code>$result = array_merge($arrArray1, $arrArray2);

$result2 = array();

foreach($result as $key => $value){
        if(array_key_exists($value['product_id'], $result2)){
               $result2[$value['product_id']]['usage'] +=  $value['usage'];
        } else{
            $result2[$value['product_id']] = $value;
        }
}

print_r($result2);
</code>```

If you want to reset your resultant array indexes use array_merge again like this

```<code>$result2 = array_merge($result2);
</code>```

Hope this will help

Body:
I want to merge two array's using key(product_id) and adding that values(usage).
Array 1

Array
(
    [0] => Array
        (
            [name] => Reschedule A Service
            [usage] => 1
            [product_id] => 8
        )

    [1] => Array
        (
            [name] => Adding An Image
            [usage] => 1
            [product_id] => 5
        )

    [2] => Array
        (
            [name] => Each Calendar Event
            [usage] => 1
            [product_id] => 14
        )

)

Array 2

Array
(
    [0] => Array
        (
            [name] => Adding An Image
            [usage] => 1
            [product_id] => 5
        )

    [1] => Array
        (
            [name] => Schedule A Service
            [usage] => 3
            [product_id] => 11
        )

    [2] => Array
        (
            [name] => Each Calendar Event
            [usage] => 2
            [product_id] => 14
        )

    [3] => Array
        (
            [name] => Sales Performance Dashboard
            [usage] => 2
            [product_id] => 30
        )

    [4] => Array
        (
            [name] => Quote
            [usage] => 1
            [product_id] => 32
        )

)

I need an out put like this merging and adding usage values.

Array
(
    [0] => Array
        (
            [name] => Adding An Image
            [usage] => 2
            [product_id] => 5
        )

    [1] => Array
        (
            [name] => Schedule A Service
            [usage] => 3
            [product_id] => 11
        )

    [2] => Array
        (
            [name] => Each Calendar Event
            [usage] => 3
            [product_id] => 14
        )

    [3] => Array
        (
            [name] => Sales Performance Dashboard
            [usage] => 2
            [product_id] => 30
        )

    [4] => Array
        (
            [name] => Quote
            [usage] => 1
            [product_id] => 32
        )

    [5] => Array
        (
            [name] => Reschedule A Service
            [usage] => 1
            [product_id] => 8
        )

)

This is my code for creating arrays

    foreach($query->rows as $product){

     $top_products[]=array(
        'name'=>$product['name'],
        'usage'=>$product['pusage'],
        'product_id'=>$product['product_id']
     );

    }

    foreach($query_2->rows as $product){

         $top_point_products[]=array(
           'name'=>$product['name'],
           'usage'=>$product['p_usage'],
           'product_id'=>$product['product_id']
         );

    }

Body:
Use array_merge and a simple foreach loop to check your condition and update the usagevalues.

See below

$result = array_merge($arrArray1, $arrArray2);

$result2 = array();

foreach($result as $key => $value){
        if(array_key_exists($value['product_id'], $result2)){
               $result2[$value['product_id']]['usage'] +=  $value['usage'];
        } else{
            $result2[$value['product_id']] = $value;
        }
}

print_r($result2);

If you want to reset your resultant array indexes use array_merge again like this

$result2 = array_merge($result2);

Hope this will help