title: Tuning Mysql MyISAM tables and queries
body: <p><strong>Folks</strong></p>
  
  <p>I have tables ( with their structures) </p>
  
  <p>Billing_datas_new has about 50 million records and it increases on daily basis. ( the call detail records) are dumped in this table along with the required ids) </p>
  
  <pre><code>CREATE TABLE IF NOT EXISTS `cctonumbers` (
    `id` int(11) NOT NULL AUTO_INCREMENT,
    `countrycode_id` int(11) NOT NULL,
    `parentid` int(11) NOT NULL,
    `number` char(20) NOT NULL,
    `dialcode` int(11) NOT NULL,
    `effectivedate` datetime NOT NULL,
    `route_code` tinyint(4) NOT NULL,
    `active` tinyint(4) NOT NULL,
    `status` tinyint(11) NOT NULL,
    `createdby` int(11) NOT NULL,
    `createdon` datetime NOT NULL,
    `modifiedby` int(11) NOT NULL,
    `modifiedon` datetime NOT NULL,
    PRIMARY KEY (`id`),
    KEY `effectivedate` (`effectivedate`),
    KEY `parentid` (`parentid`),
    KEY `number` (`number`),
    KEY `dialcode` (`dialcode`)
  ) ENGINE=InnoDB  DEFAULT CHARSET=latin1 AUTO_INCREMENT=15729 ;
  
  
  CREATE TABLE IF NOT EXISTS `billing_datas_NEW` (
    `id` int(11) NOT NULL AUTO_INCREMENT,
    `batch_id` int(11) NOT NULL,
    `file_url` varchar(255) NOT NULL,
    `switch_id` int(11) NOT NULL,
    `carrierid_supplier` int(11) NOT NULL,
    `technical_profileid_supplier` int(11) NOT NULL,
    `carrierid_customer` int(11) NOT NULL,
    `technical_profileid_customer` int(11) NOT NULL,
    `billing_increment_supplier` varchar(10) NOT NULL,
    `billing_increment_customer` varchar(10) NOT NULL,
    `billable_duration_supplier` int(11) NOT NULL,
    `call_duration_seconds` int(11) NOT NULL,
    `billable_duration_customer` int(11) NOT NULL,
    `destination` varchar(50) NOT NULL,
    `destination_country` varchar(50) NOT NULL,
    `rateplanid_supplier` int(11) NOT NULL,
    `rateplanid_customer` int(11) NOT NULL,
    `rate_supplier` int(11) NOT NULL,
    `rate_customer` int(11) NOT NULL,
    `rate_total_supplier` int(11) NOT NULL,
    `rate_total_customer` int(11) NOT NULL,
    `rate_effective_date_supplier` date NOT NULL,
    `rate_effective_date_customer` date NOT NULL,
    `call_hour` varchar(10) NOT NULL,
    `sequence_number` int(11) NOT NULL,
    `version` varchar(10) NOT NULL,
    `record_type` varchar(1) NOT NULL,
    `connection_type` varchar(16) NOT NULL,
    `session_id` varchar(36) NOT NULL,
    `release_cause` smallint(6) NOT NULL,
    `start_time_date` datetime NOT NULL,
    `answer_time_date` datetime NOT NULL,
    `release_tod` datetime NOT NULL,
    `greenwich_mean_time` varchar(32) NOT NULL,
    `release_cause_protocol_stack` varchar(32) NOT NULL,
    `binary_value_protocol_stack` smallint(6) NOT NULL,
    `first_release_dialogue` varchar(1) NOT NULL,
    `origination_trunkid` int(11) NOT NULL,
    `voip_protocol` varchar(6) NOT NULL,
    `source_number` varchar(128) NOT NULL,
    `source_host_name` varchar(128) NOT NULL,
    `destination_number` varchar(128) NOT NULL,
    `destination_host_name` varchar(128) NOT NULL,
    `callid` varchar(128) NOT NULL,
    `remote_payload_ipaddress` varchar(16) NOT NULL,
    `remote_payload_udpaddress` varchar(6) NOT NULL,
    `local_payload_ipaddress` varchar(16) NOT NULL,
    `local_payload_udpaddress` varchar(6) NOT NULL,
    `codec_list` varchar(128) NOT NULL,
    `ingress_packets` int(11) NOT NULL,
    `egress_packets` int(11) NOT NULL,
    `ingress_octets` int(11) NOT NULL,
    `egress_octets` int(11) NOT NULL,
    `ingress_packet_loss` int(11) NOT NULL,
    `ingress_delay` int(11) NOT NULL,
    `ingress_packet_jitter` int(11) NOT NULL,
    `supplierid` mediumint(9) NOT NULL,
    `protocol` varchar(6) NOT NULL,
    `termination_source_number` varchar(128) NOT NULL,
    `termination_source_host` varchar(128) NOT NULL,
    `termination_destination_number` varchar(128) NOT NULL,
    `termination_destination_host_name` varchar(128) NOT NULL,
    `termination_callid` varchar(128) NOT NULL,
    `termination_remote_payload_ipaddress` varchar(16) NOT NULL,
    `termination_remote_payload_udpaddress` varchar(6) NOT NULL,
    `termination_local_payload_ipaddress` varchar(16) NOT NULL,
    `termination_local_payload_udpaddress` varchar(6) NOT NULL,
    `termination_codec_list` varchar(128) NOT NULL,
    `termination_ingress_packets` int(11) NOT NULL,
    `termination_egress_packets` int(11) NOT NULL,
    `termination_ingress_octets` int(11) NOT NULL,
    `termination_egress_octets` int(11) NOT NULL,
    `termination_ingress_packet_loss` int(11) NOT NULL,
    `termination_ingress_delay` int(11) NOT NULL,
    `termination_ingress_packet_jitter` int(11) NOT NULL,
    `final_route_indication` varchar(1) NOT NULL,
    `routing_digits` varchar(64) NOT NULL,
    `call_duration` mediumint(9) NOT NULL,
    `post_dial_delay` mediumint(9) NOT NULL,
    `ring_time` mediumint(9) NOT NULL,
    `call_duration_ms` int(11) NOT NULL,
    `confid` varchar(32) NOT NULL,
    `rpid` varchar(32) NOT NULL,
    `route_entry_index` tinyint(4) NOT NULL,
    `route_table_used` mediumint(9) NOT NULL,
    `lnp_dipped` varchar(1) NOT NULL,
    `ingress_lrn` varchar(32) NOT NULL,
    `egress_lrn` varchar(32) NOT NULL,
    `cnam_dipped` tinyint(4) NOT NULL,
    `dnc_dipped` tinyint(4) NOT NULL,
    `origination_device_name` varchar(15) NOT NULL,
    `termination_device_name` varchar(15) NOT NULL,
    `ers_dipped` varchar(1) NOT NULL,
    `oli_digits` varchar(8) NOT NULL,
    `routing_match` varchar(41) NOT NULL,
    `termination_pdd` varchar(6) NOT NULL,
    `proxy_charge_information` varchar(41) NOT NULL,
    `jurisdictoinal_indication_parameter` varchar(41) NOT NULL,
    `matched_digits_ers` varchar(41) NOT NULL,
    `route_table_irs` varchar(5) NOT NULL,
    `route_sequence_number_ers` varchar(2) NOT NULL,
    `jurisdiction_type_ers` varchar(3) NOT NULL,
    PRIMARY KEY (`id`),
    KEY `switch_id` (`switch_id`),
    KEY `carrieridsupplier` (`carrierid_supplier`),
    KEY `technicalprofileidsupplier` (`technical_profileid_supplier`),
    KEY `carrieridcustomer` (`carrierid_customer`),
    KEY `technicalprofileidcustomer` (`technical_profileid_customer`),
    KEY `destination` (`destination`),
    KEY `destinationcountry` (`destination_country`),
    KEY `rateplanidsupplier` (`rateplanid_supplier`),
    KEY `rateplanidcustomer` (`rateplanid_customer`),
    KEY `callhour` (`call_hour`),
    KEY `recordtype` (`record_type`),
    KEY `connectiontype` (`connection_type`),
    KEY `sessionid` (`session_id`),
    KEY `releasecause` (`release_cause`),
    KEY `starttimedate` (`start_time_date`),
    KEY `protocolstack` (`release_cause_protocol_stack`),
    KEY `binaryvalue` (`binary_value_protocol_stack`),
    KEY `originationtrunkid` (`origination_trunkid`),
    KEY `sourcehostname` (`source_host_name`),
    KEY `destinationhostname` (`destination_host_name`),
    KEY `supplierid` (`supplierid`),
    KEY `terminationsourcehost` (`termination_source_host`),
    KEY `terminationdestinationhostname` (`termination_destination_host_name`),
    KEY `callduration` (`call_duration`),
    KEY `postdialdelay` (`post_dial_delay`),
    KEY `batch_id` (`batch_id`)
  ) ENGINE=MyISAM  DEFAULT CHARSET=latin1 AUTO_INCREMENT=56232766 ;
  
  
  CREATE TABLE IF NOT EXISTS `carriermasters` (
    `id` int(11) NOT NULL AUTO_INCREMENT,
    `carriername` varchar(255) NOT NULL,
    `carriershortname` varchar(20) NOT NULL,
    `website` varchar(255) NOT NULL,
    `phone` int(11) NOT NULL,
    `fax` int(11) NOT NULL,
    `email` varchar(255) NOT NULL,
    `billingaddress` varchar(255) NOT NULL,
    `billingcity` varchar(200) NOT NULL,
    `billingstate` varchar(200) NOT NULL,
    `billingcountry` varchar(200) NOT NULL,
    `zipcode` int(11) DEFAULT NULL,
    `description` varchar(255) NOT NULL,
    `minimumseconds` int(11) NOT NULL,
    `incrementseconds` int(11) NOT NULL,
    `active` tinyint(4) NOT NULL DEFAULT '1',
    `createdby` int(11) NOT NULL,
    `createdon` datetime NOT NULL,
    `modifiedby` int(11) NOT NULL,
    `modifiedon` datetime NOT NULL,
    `account_manager` varchar(500) NOT NULL,
    `status` varchar(100) NOT NULL,
    PRIMARY KEY (`id`),
    KEY `carriershortname` (`carriershortname`)
  ) ENGINE=InnoDB  DEFAULT CHARSET=latin1 AUTO_INCREMENT=306 ;
  
  
  CREATE TABLE IF NOT EXISTS `technicalprofiles` (
    `id` int(11) NOT NULL AUTO_INCREMENT,
    `carrier_id` int(11) NOT NULL,
    `profilename` varchar(255) NOT NULL,
    `active` tinyint(4) NOT NULL DEFAULT '1',
    `createdby` int(50) NOT NULL,
    `createdon` datetime NOT NULL,
    `modifiedby` int(50) NOT NULL,
    `modifiedon` datetime NOT NULL,
    PRIMARY KEY (`id`),
    KEY `carrier_id` (`carrier_id`),
    KEY `profilename` (`profilename`)
  ) ENGINE=InnoDB  DEFAULT CHARSET=latin1 AUTO_INCREMENT=479 ;
  </code></pre>
  
  <p>MY.CNF for the server is as below </p>
  
  <pre><code># Example MySQL config file for very large systems.
  #
  # This is for a large system with memory of 1G-2G where the system runs mainly
  # MySQL.
  #
  # MySQL programs look for option files in a set of
  # locations which depend on the deployment platform.
  # You can copy this option file to one of those
  # locations. For information about these locations, see:
  # http://dev.mysql.com/doc/mysql/en/option-files.html
  #
  # In this file, you can use all long options that a program supports.
  # If you want to know which options a program supports, run the program
  # with the "--help" option.
  
  # The following options will be passed to all MySQL clients
  [client]
  #password   = your_password
  port        = 3306
  socket      = /var/lib/mysql/mysql.sock
  
  # Here follows entries for some specific programs
  
  # The MySQL server
  [mysqld]
  port        = 3306
  socket      = /var/lib/mysql/mysql.sock
  skip-locking
  key_buffer_size = 3G
  max_allowed_packet = 16M
  table_open_cache = 1024
  sort_buffer_size = 256M
  read_buffer_size = 1024M
  read_rnd_buffer_size = 64M
  myisam_sort_buffer_size = 4G
  thread_cache_size = 8
  query_cache_size = 256M
  query_cache_limit = 256M
  
  max_connections = 100   
  max_heap_table_size = 64M
  join_buffer_size = 8M
  tmp_table_size = 1024M
  myisam_max_sort_file_size = 3G
  slow_query_log
  
  
  
  # Try number of CPU's*2 for thread_concurrency
  thread_concurrency = 8
  
  # Don't listen on a TCP/IP port at all. This can be a security enhancement,
  # if all processes that need to connect to mysqld run on the same host.
  # All interaction with mysqld must be made via Unix sockets or named pipes.
  # Note that using this option without enabling named pipes on Windows
  # (via the "enable-named-pipe" option) will render mysqld useless!
  #
  #skip-networking
  
  # Replication Master Server (default)
  # binary logging is required for replication
  log-bin=mysql-bin
  
  # required unique id between 1 and 2^32 - 1
  # defaults to 1 if master-host is not set
  # but will not function as a master if omitted
  server-id   = 1
  
  # Replication Slave (comment out master section to use this)
  #
  # To configure this host as a replication slave, you can choose between
  # two methods :
  #
  # 1) Use the CHANGE MASTER TO command (fully described in our manual) -
  #    the syntax is:
  #
  #    CHANGE MASTER TO MASTER_HOST=&lt;host&gt;, MASTER_PORT=&lt;port&gt;,
  #    MASTER_USER=&lt;user&gt;, MASTER_PASSWORD=&lt;password&gt; ;
  #
  #    where you replace &lt;host&gt;, &lt;user&gt;, &lt;password&gt; by quoted strings and
  #    &lt;port&gt; by the master's port number (3306 by default).
  #
  #    Example:
  #
  #    CHANGE MASTER TO MASTER_HOST='125.564.12.1', MASTER_PORT=3306,
  #    MASTER_USER='joe', MASTER_PASSWORD='secret';
  #
  # OR
  #
  # 2) Set the variables below. However, in case you choose this method, then
  #    start replication for the first time (even unsuccessfully, for example
  #    if you mistyped the password in master-password and the slave fails to
  #    connect), the slave will create a master.info file, and any later
  #    change in this file to the variables' values below will be ignored and
  #    overridden by the content of the master.info file, unless you shutdown
  #    the slave server, delete master.info and restart the slaver server.
  #    For that reason, you may want to leave the lines below untouched
  #    (commented) and instead use CHANGE MASTER TO (see above)
  #
  # required unique id between 2 and 2^32 - 1
  # (and different from the master)
  # defaults to 2 if master-host is set
  # but will not function as a slave if omitted
  #server-id       = 2
  #
  # The replication master for this slave - required
  #master-host     =   &lt;hostname&gt;
  #
  # The username the slave will use for authentication when connecting
  # to the master - required
  #master-user     =   &lt;username&gt;
  #
  # The password the slave will authenticate with when connecting to
  # the master - required
  #master-password =   &lt;password&gt;
  #
  # The port the master is listening on.
  # optional - defaults to 3306
  #master-port     =  &lt;port&gt;
  #
  # binary logging - not required for slaves, but recommended
  #log-bin=mysql-bin
  #
  # binary logging format - mixed recommended
  #binlog_format=mixed
  
  # Uncomment the following if you are using InnoDB tables
  #innodb_data_home_dir = /var/lib/mysql
  #innodb_data_file_path = ibdata1:2000M;ibdata2:10M:autoextend
  #innodb_log_group_home_dir = /var/lib/mysql
  # You can set .._buffer_pool_size up to 50 - 80 %
  # of RAM but beware of setting memory usage too high
  #innodb_buffer_pool_size = 384M
  #innodb_additional_mem_pool_size = 20M
  # Set .._log_file_size to 25 % of buffer pool size
  #innodb_log_file_size = 100M
  #innodb_log_buffer_size = 8M
  #innodb_flush_log_at_trx_commit = 1
  #innodb_lock_wait_timeout = 50
  
  [mysqldump]
  quick
  max_allowed_packet = 100M
  
  [mysql]
  no-auto-rehash
  # Remove the next comment character if you are not familiar with SQL
  #safe-updates
  
  [myisamchk]
  key_buffer_size = 3G
  sort_buffer_size = 512M
  read_buffer = 1024M
  write_buffer = 8M
  
  [mysqlhotcopy]
  interactive-timeout
  </code></pre>
  
  <p>mysql tuner report is  as below</p>
  
  <pre><code>-- MYSQL PERFORMANCE TUNING PRIMER --
           - By: Matthew Montgomery -
  
  MySQL Version 5.1.69-log x86_64
  
  Uptime = 1 days 9 hrs 7 min 0 sec
  Avg. qps = 13
  Total Questions = 1616144
  Threads Connected = 2
  
  Warning: Server has not been running for at least 48hrs.
  It may not be safe to use these recommendations
  
  To find out more information on how each of these
  runtime variables effects performance visit:
  http://dev.mysql.com/doc/refman/5.1/en/server-system-variables.html
  Visit http://www.mysql.com/products/enterprise/advisors.html
  for info about MySQL's Enterprise Monitoring and Advisory Service
  
  SLOW QUERIES
  The slow query log is enabled.
  Current long_query_time = 10.000000 sec.
  You have 30 out of 1616167 that take longer than 10.000000 sec. to complete
  Your long_query_time seems to be fine
  
  BINARY UPDATE LOG
  The binary update log is enabled
  The expire_logs_days is not set.
  The mysqld will retain the entire binary log until RESET MASTER or PURGE MASTER LOGS commands are run manually
  Setting expire_logs_days will allow you to remove old binary logs automatically
  See http://dev.mysql.com/doc/refman/5.1/en/purge-master-logs.html
  Binlog sync is not enabled, you could loose binlog records during a server crash
  
  WORKER THREADS
  Current thread_cache_size = 8
  Current threads_cached = 6
  Current threads_per_sec = 0
  Historic threads_per_sec = 0
  Your thread_cache_size is fine
  
  MAX CONNECTIONS
  Current max_connections = 100
  Current threads_connected = 2
  Historic max_used_connections = 9
  The number of used connections is 9% of the configured maximum.
  You are using less than 10% of your configured max_connections.
  Lowering max_connections could help to avoid an over-allocation of memory
  See "MEMORY USAGE" section to make sure you are not over-allocating
  
  INNODB STATUS
  Current InnoDB index space = 3 M
  Current InnoDB data space = 20 M
  Current InnoDB buffer pool free = 0 %
  Current innodb_buffer_pool_size = 8 M
  Depending on how much space your innodb indexes take up it may be safe
  to increase this value to up to 2 / 3 of total system memory
  
  MEMORY USAGE
  Max Memory Ever Allocated : 15.14 G
  Configured Max Per-thread Buffers : 132.05 G
  Configured Max Global Buffers : 3.25 G
  Configured Max Memory Limit : 135.31 G
  Physical Memory : 7.68 G
  
  Max memory limit exceeds 90% of physical memory
  
  KEY BUFFER
  Current MyISAM index space = 16.92 G
  Current key_buffer_size = 3.00 G
  Key cache miss rate is 1 : 2326
  Key buffer free ratio = 0 %
  You could increase key_buffer_size
  It is safe to raise this up to 1/4 of total system memory;
  assuming this is a dedicated database server.
  
  QUERY CACHE
  Query cache is enabled
  Current query_cache_size = 256 M
  Current query_cache_used = 2 M
  Current query_cache_limit = 256 M
  Current Query cache Memory fill ratio = .94 %
  Current query_cache_min_res_unit = 4 K
  Your query_cache_size seems to be too high.
  Perhaps you can use these resources elsewhere
  MySQL won't cache query results that are larger than query_cache_limit in size
  
  SORT OPERATIONS
  Current sort_buffer_size = 256 M
  Current read_rnd_buffer_size = 64 M
  Sort buffer seems to be fine
  
  JOINS
  Current join_buffer_size = 8.00 M
  You have had 408 queries where a join could not use an index properly
  You have had 5836 joins without keys that check for key usage after each row
  join_buffer_size &gt;= 4 M
  This is not advised
  You should enable "log-queries-not-using-indexes"
  Then look for non indexed joins in the slow query log.
  
  OPEN FILES LIMIT
  Current open_files_limit = 2158 files
  The open_files_limit should typically be set to at least 2x-3x
  that of table_cache if you have heavy MyISAM usage.
  Your open_files_limit value seems to be fine
  
  TABLE CACHE
  Current table_open_cache = 1024 tables
  Current table_definition_cache = 256 tables
  You have a total of 119 tables
  You have 175 open tables.
  The table_cache value seems to be fine
  
  TEMP TABLES
  Current max_heap_table_size = 64 M
  Current tmp_table_size = 1.00 G
  Of 209033 temp tables, 7% were created on disk
  Effective in-memory tmp_table_size is limited to max_heap_table_size.
  Created disk tmp tables ratio seems fine
  
  TABLE SCANS
  Current read_buffer_size = 1.00 G
  Current table scan ratio = 35358 : 1
  read_buffer_size is over 8 MB there is probably no need for such a large read_buffer
  
  TABLE LOCKING
  Current Lock Wait ratio = 1 : 61064
  Your table locking seems to be fine
  </code></pre>
  
  <p>The stored procedure is as attached</p>
  
  <pre><code>BEGIN
  
  create temporary table if not exists temp_cns (id int primary key, countryname varchar(255));
  insert into temp_cns select c1.id, c2.countryname from cctonumbers c1 left join countrycodes c2 on c1.countrycode_id = c2.id;
  
  
  drop temporary table if exists temp_cause_code_id;
  
  SET @temp_ids = CONCAT('create temporary table temp_cause_code_id select id from billing_datas_NEW use index(primary) where ', conditions_str);
  PREPARE stmt FROM @temp_ids;
  EXECUTE stmt;
  DEALLOCATE PREPARE stmt;
  
  drop temporary table if exists temp_cause_code_allfileds;
  
  create temporary table temp_cause_code_allfileds select m_tbl.id,m_tbl.destination,(select dialcode from cctonumbers where id = m_tbl.destination) as dial_code,(select number from cctonumbers where id = m_tbl.destination_country) as country_value,m_tbl.destination_country,m_tbl.release_cause_protocol_stack, m_tbl.binary_value_protocol_stack, m_tbl.release_cause,m_tbl.call_duration_seconds, m_tbl.destination_country as destination_name,m_tbl.start_time_date,m_tbl.start_time_date as start_date,m_tbl.carrierid_customer,m_tbl.technical_profileid_customer,m_tbl.origination_trunkid,m_tbl.carrierid_supplier,m_tbl.technical_profileid_supplier,m_tbl.supplierid,cm.carriername as customer,ctech.profilename as customer_tech,(select carriername from carriermasters where id = carrierid_supplier) as vendor,(select profilename from technicalprofiles where id = technical_profileid_supplier) as vendor_tech_profile from temp_cause_code_id as temp_ids LEFT JOIN billing_datas_NEW  as m_tbl  ON temp_ids.id = m_tbl.id LEFT JOIN carriermasters as cm ON m_tbl.carrierid_customer = cm.id LEFT JOIN technicalprofiles as ctech ON m_tbl.technical_profileid_customer = ctech.id WHERE m_tbl.id IS NOT NULL;
  
  drop temporary table if exists temp_group_records; 
  
  SET @sqlv = CONCAT('create temporary table temp_group_records select destination,dial_code,country_value, start_date, destination_country, release_cause_protocol_stack, binary_value_protocol_stack, release_cause, destination_name, count(*) as calls, (sum(call_duration_seconds) / 60) as durations, sum(call_duration_seconds &gt; 0) as scalls,customer,customer_tech,vendor,vendor_tech_profile from temp_cause_code_allfileds  group by binary_value_protocol_stack, release_cause_protocol_stack, destination, destination_country ');
  PREPARE stmt FROM @sqlv;
  EXECUTE stmt;
  DEALLOCATE PREPARE stmt;
  
  update temp_group_records set destination = destination_country where destination = '';
  
  update temp_group_records t, temp_cns c set t.destination_name = c.countryname where t.destination = c.id;
  
  drop table if exists temp_records;
  
  create table temp_records select customer,customer_tech,vendor,vendor_tech_profile,release_cause_protocol_stack, binary_value_protocol_stack,dial_code,country_value, release_cause, destination_name, sum(calls) as ncalls, sum(durations) as ndurations, sum(scalls) as nscalls,start_date from temp_group_records group by binary_value_protocol_stack, release_cause_protocol_stack, destination_name;
  
  END
  </code></pre>
  
  <p>The condition_str in the above stored procedure along with its explain in mysql is in the link as below</p>
  
  <p><a href="http://www.prosesindia.com/dock.docx" rel="nofollow">http://www.prosesindia.com/dock.docx</a></p>
  
  <p>The problem is </p>
  
  <p>when i run the above stored procedure , it takes about 1 min and 40 seconds. </p>
  
  <p>is there any way we can tune the stored procedure or the server variables. </p>